diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae61fd9f8aa..92b35e97baa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ attention. --> - Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)? -- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/.github/CONTRIBUTING.md)? +- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md)? - If submitting code, have you built your formula locally prior to submission with `gradle check`? - If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed. - If submitting code, have you checked that your submission is for an [OS that we support](https://www.elastic.co/support/matrix#show_os)? diff --git a/TESTING.asciidoc b/TESTING.asciidoc index ce81f97548f..44eda08020a 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -296,7 +296,7 @@ gradle :distribution:integ-test-zip:integTest \ -Dtests.method="test {p0=cat.shards/10_basic/Help}" --------------------------------------------------------------------------- -`RestNIT` are the executable test classes that runs all the +`RestIT` are the executable test classes that runs all the yaml suites available within the `rest-api-spec` folder. The REST tests support all the options provided by the randomized runner, plus the following: diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000000..afe60c89368 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,63 @@ +# Elasticsearch Microbenchmark Suite + +This directory contains the microbenchmark suite of Elasticsearch. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/). + +## Purpose + +We do not want to microbenchmark everything but the kitchen sink and should typically rely on our +[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with +[Rally](http://github.com/elastic/rally). Microbenchmarks are intended for performance-critical components to spot performance +regressions. The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR. + +## Getting Started + +Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result. + +## Running Microbenchmarks + +Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`. +``` + +Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks). + +If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have any other special requirements +generate the uberjar with `gradle :benchmarks:jmhJar` and run the it directly with: + +``` +java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar +``` + +JMH supports lots of command line parameters. Add `-h` to the command above for more information about the available command line options. + +## Adding Microbenchmarks + +Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the +[JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). + +In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and +end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. + +## Tips and Best Practices + +To get realistic results, you should exercise care when running your benchmarks. Here are a few tips: + +### Do + +* Ensure that the system executing your microbenchmarks has as little load as possible and shutdown every process that can cause unnecessary + runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance. +* Ensure to run enough warmup iterations to get into a stable state. If you are unsure, don't change the defaults. +* Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`. +* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the + `performance` CPU governor. +* Vary problem input size with `@Param`. +* Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses: +** Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews + your results. If so, try to force a GC between runs (`-gc true`). +** Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots. +* Have your benchmarks peer-reviewed. + +### Don't + +* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.e. with `-prof perfasm`. +* Run run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks). +* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable. \ No newline at end of file diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle new file mode 100644 index 00000000000..186fdca44ea --- /dev/null +++ b/benchmarks/build.gradle @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +buildscript { + repositories { + maven { + url 'https://plugins.gradle.org/m2/' + } + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3' + } +} + +apply plugin: 'elasticsearch.build' +// build an uberjar with all benchmarks +apply plugin: 'com.github.johnrengelman.shadow' +// have the shadow plugin provide the runShadow task +apply plugin: 'application' + +archivesBaseName = 'elasticsearch-benchmarks' +mainClassName = 'org.openjdk.jmh.Main' + +// never try to invoke tests on the benchmark project - there aren't any +check.dependsOn.remove(test) +// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip +task test(type: Test, overwrite: true) + +dependencies { + compile("org.elasticsearch:elasticsearch:${version}") { + // JMH ships with the conflicting version 4.6 (JMH will not update this dependency as it is Java 6 compatible and joptsimple is one + // of the most recent compatible version). This prevents us from using jopt-simple in benchmarks (which should be ok) but allows us + // to invoke the JMH uberjar as usual. + exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' + } + compile "org.openjdk.jmh:jmh-core:$versions.jmh" + compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" + // Dependencies of JMH + runtime 'net.sf.jopt-simple:jopt-simple:4.6' + runtime 'org.apache.commons:commons-math3:3.2' +} + +compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" +compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" + +forbiddenApis { + // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes + ignoreFailures = true +} + +// No licenses for our benchmark deps (we don't ship benchmarks) +dependencyLicenses.enabled = false + +thirdPartyAudit.excludes = [ + // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees) + 'org.openjdk.jmh.profile.AbstractHotspotProfiler', + 'org.openjdk.jmh.profile.HotspotThreadProfiler', + 'org.openjdk.jmh.profile.HotspotClassloadingProfiler', + 'org.openjdk.jmh.profile.HotspotCompilationProfiler', + 'org.openjdk.jmh.profile.HotspotMemoryProfiler', + 'org.openjdk.jmh.profile.HotspotRuntimeProfiler', + 'org.openjdk.jmh.util.Utils' +] + +shadowJar { + classifier = 'benchmarks' +} + +// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface +task jmhJar( + dependsOn: shadowJar, + description: 'Generates an uberjar with the microbenchmarks and all dependencies', + group: 'Benchmark' +) + +task jmh( + dependsOn: runShadow, + description: 'Runs all microbenchmarks', + group: 'Benchmark' +) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java new file mode 100644 index 00000000000..d5bb0916eca --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java @@ -0,0 +1,170 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.settings.Settings; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +@Fork(3) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@SuppressWarnings("unused") //invoked by benchmarking framework +public class AllocationBenchmark { + // Do NOT make any field final (even if it is not annotated with @Param)! See also + // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java + + // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no + // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would + // need its own main method and we cannot execute more than one class with a main method per JAR. + @Param({ + // indices, shards, replicas, nodes + " 10, 1, 0, 1", + " 10, 3, 0, 1", + " 10, 10, 0, 1", + " 100, 1, 0, 1", + " 100, 3, 0, 1", + " 100, 10, 0, 1", + + " 10, 1, 0, 10", + " 10, 3, 0, 10", + " 10, 10, 0, 10", + " 100, 1, 0, 10", + " 100, 3, 0, 10", + " 100, 10, 0, 10", + + " 10, 1, 1, 10", + " 10, 3, 1, 10", + " 10, 10, 1, 10", + " 100, 1, 1, 10", + " 100, 3, 1, 10", + " 100, 10, 1, 10", + + " 10, 1, 2, 10", + " 10, 3, 2, 10", + " 10, 10, 2, 10", + " 100, 1, 2, 10", + " 100, 3, 2, 10", + " 100, 10, 2, 10", + + " 10, 1, 0, 50", + " 10, 3, 0, 50", + " 10, 10, 0, 50", + " 100, 1, 0, 50", + " 100, 3, 0, 50", + " 100, 10, 0, 50", + + " 10, 1, 1, 50", + " 10, 3, 1, 50", + " 10, 10, 1, 50", + " 100, 1, 1, 50", + " 100, 3, 1, 50", + " 100, 10, 1, 50", + + " 10, 1, 2, 50", + " 10, 3, 2, 50", + " 10, 10, 2, 50", + " 100, 1, 2, 50", + " 100, 3, 2, 50", + " 100, 10, 2, 50" + }) + public String indicesShardsReplicasNodes = "10,1,0,1"; + + public int numTags = 2; + + private AllocationService strategy; + private ClusterState initialClusterState; + + @Setup + public void setUp() throws Exception { + final String[] params = indicesShardsReplicasNodes.split(","); + + int numIndices = toInt(params[0]); + int numShards = toInt(params[1]); + int numReplicas = toInt(params[2]); + int numNodes = toInt(params[3]); + + strategy = Allocators.createAllocationService(Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "tag") + .build()); + + MetaData.Builder mb = MetaData.builder(); + for (int i = 1; i <= numIndices; i++) { + mb.put(IndexMetaData.builder("test_" + i) + .settings(Settings.builder().put("index.version.created", Version.CURRENT)) + .numberOfShards(numShards) + .numberOfReplicas(numReplicas) + ); + } + MetaData metaData = mb.build(); + RoutingTable.Builder rb = RoutingTable.builder(); + for (int i = 1; i <= numIndices; i++) { + rb.addAsNew(metaData.index("test_" + i)); + } + RoutingTable routingTable = rb.build(); + DiscoveryNodes.Builder nb = DiscoveryNodes.builder(); + for (int i = 1; i <= numNodes; i++) { + nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags)))); + } + initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes + (nb).build(); + } + + private int toInt(String v) { + return Integer.valueOf(v.trim()); + } + + @Benchmark + public ClusterState measureAllocation() { + ClusterState clusterState = initialClusterState; + while (clusterState.getRoutingNodes().hasUnassignedShards()) { + RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes() + .shardsWithState(ShardRoutingState.INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + result = strategy.reroute(clusterState, "reroute"); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + } + return clusterState; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java new file mode 100644 index 00000000000..aba7fda1021 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.benchmark.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.gateway.GatewayAllocator; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class Allocators { + private static class NoopGatewayAllocator extends GatewayAllocator { + public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator(); + + protected NoopGatewayAllocator() { + super(Settings.EMPTY, null, null); + } + + @Override + public void applyStartedShards(StartedRerouteAllocation allocation) { + // noop + } + + @Override + public void applyFailedShards(FailedRerouteAllocation allocation) { + // noop + } + + @Override + public boolean allocateUnassigned(RoutingAllocation allocation) { + return false; + } + } + + private Allocators() { + throw new AssertionError("Do not instantiate"); + } + + + public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException, + IllegalAccessException, InvocationTargetException { + return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings + .BUILT_IN_CLUSTER_SETTINGS)); + } + + public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws + InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + return new AllocationService(settings, + defaultAllocationDeciders(settings, clusterSettings), + NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE); + } + + public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws + IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException { + List list = new ArrayList<>(); + // Keep a deterministic order of allocation deciders for the benchmark + for (Class deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) { + try { + Constructor constructor = deciderClass.getConstructor(Settings.class, ClusterSettings + .class); + list.add(constructor.newInstance(settings, clusterSettings)); + } catch (NoSuchMethodException e) { + Constructor constructor = deciderClass.getConstructor(Settings.class); + list.add(constructor.newInstance(settings)); + } + } + return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0])); + + } + + public static DiscoveryNode newNode(String nodeId, Map attributes) { + return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER, + DiscoveryNode.Role.DATA), Version.CURRENT); + } +} diff --git a/benchmarks/src/main/resources/log4j.properties b/benchmarks/src/main/resources/log4j.properties new file mode 100644 index 00000000000..8ca1bc87295 --- /dev/null +++ b/benchmarks/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Do not log at all if it is not really critical - we're in a benchmark +benchmarks.es.logger.level=ERROR +log4j.rootLogger=${benchmarks.es.logger.level}, out + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n + diff --git a/build.gradle b/build.gradle index 5c6ec366930..e78876a84c7 100644 --- a/build.gradle +++ b/build.gradle @@ -160,6 +160,7 @@ subprojects { them as external dependencies so the build plugin that we use can be used to build elasticsearch plugins outside of the elasticsearch source tree. */ ext.projectSubstitutions = [ + "org.elasticsearch.gradle:build-tools:${version}": ':build-tools', "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec', "org.elasticsearch:elasticsearch:${version}": ':core', "org.elasticsearch.test:framework:${version}": ':test:framework', diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 623fdab3e3e..a5a9b4209ba 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -103,6 +103,7 @@ if (project == rootProject) { url "https://oss.sonatype.org/content/repositories/snapshots/" } } + test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*' } /***************************************************************************** @@ -122,8 +123,8 @@ if (project != rootProject) { // build-tools is not ready for primetime with these... dependencyLicenses.enabled = false forbiddenApisMain.enabled = false + forbiddenApisTest.enabled = false jarHell.enabled = false - loggerUsageCheck.enabled = false thirdPartyAudit.enabled = false // test for elasticsearch.build tries to run with ES... @@ -137,4 +138,9 @@ if (project != rootProject) { // the file that actually defines nocommit exclude '**/ForbiddenPatternsTask.groovy' } + + namingConventions { + testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase' + integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase' + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy index 612bc568621..fe7f13f29e6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy @@ -21,6 +21,7 @@ package org.elasticsearch.gradle.precommit import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.VersionProperties +import org.gradle.api.artifacts.Dependency import org.gradle.api.file.FileCollection import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles @@ -57,8 +58,27 @@ public class NamingConventionsTask extends LoggedExec { @Input boolean skipIntegTestInDisguise = false + /** + * Superclass for all tests. + */ + @Input + String testClass = 'org.apache.lucene.util.LuceneTestCase' + + /** + * Superclass for all integration tests. + */ + @Input + String integTestClass = 'org.elasticsearch.test.ESIntegTestCase' + public NamingConventionsTask() { - dependsOn(classpath) + // Extra classpath contains the actual test + project.configurations.create('namingConventions') + Dependency buildToolsDep = project.dependencies.add('namingConventions', + "org.elasticsearch.gradle:build-tools:${VersionProperties.elasticsearch}") + buildToolsDep.transitive = false // We don't need gradle in the classpath. It conflicts. + FileCollection extraClasspath = project.configurations.namingConventions + dependsOn(extraClasspath) + description = "Runs NamingConventionsCheck on ${classpath}" executable = new File(project.javaHome, 'bin/java') onlyIf { project.sourceSets.test.output.classesDir.exists() } @@ -69,7 +89,8 @@ public class NamingConventionsTask extends LoggedExec { project.afterEvaluate { doFirst { args('-Djna.nosys=true') - args('-cp', classpath.asPath, 'org.elasticsearch.test.NamingConventionsCheck') + args('-cp', (classpath + extraClasspath).asPath, 'org.elasticsearch.test.NamingConventionsCheck') + args(testClass, integTestClass) if (skipIntegTestInDisguise) { args('--skip-integ-tests-in-disguise') } @@ -79,7 +100,7 @@ public class NamingConventionsTask extends LoggedExec { * process of ignoring them lets us validate that they were found so this ignore parameter acts * as the test for the NamingConventionsCheck. */ - if (':test:framework'.equals(project.path)) { + if (':build-tools'.equals(project.path)) { args('--self-test') } args('--', project.sourceSets.test.output.classesDir.absolutePath) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index 48a4d7c26dc..a5e1e4c8932 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -34,7 +34,6 @@ class PrecommitTasks { configureForbiddenApis(project), configureCheckstyle(project), configureNamingConventions(project), - configureLoggerUsage(project), project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class), project.tasks.create('licenseHeaders', LicenseHeadersTask.class), project.tasks.create('jarHell', JarHellTask.class), @@ -49,6 +48,20 @@ class PrecommitTasks { UpdateShasTask updateShas = project.tasks.create('updateShas', UpdateShasTask.class) updateShas.parentTask = dependencyLicenses } + if (project.path != ':build-tools') { + /* + * Sadly, build-tools can't have logger-usage-check because that + * would create a circular project dependency between build-tools + * (which provides NamingConventionsCheck) and :test:logger-usage + * which provides the logger usage check. Since the build tools + * don't use the logger usage check because they don't have any + * of Elaticsearch's loggers and :test:logger-usage actually does + * use the NamingConventionsCheck we break the circular dependency + * here. + */ + precommitTasks.add(configureLoggerUsage(project)) + } + Map precommitOptions = [ name: 'precommit', diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 83dd1b7e4c5..c3004a64b86 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -291,9 +291,10 @@ class ClusterFormationTasks { File configDir = new File(node.homeDir, 'config') copyConfig.into(configDir) // copy must always have a general dest dir, even though we don't use it for (Map.Entry extraConfigFile : node.config.extraConfigFiles.entrySet()) { + Object extraConfigFileValue = extraConfigFile.getValue() copyConfig.doFirst { // make sure the copy won't be a no-op or act on a directory - File srcConfigFile = project.file(extraConfigFile.getValue()) + File srcConfigFile = project.file(extraConfigFileValue) if (srcConfigFile.isDirectory()) { throw new GradleException("Source for extraConfigFile must be a file: ${srcConfigFile}") } @@ -303,7 +304,7 @@ class ClusterFormationTasks { } File destConfigFile = new File(node.homeDir, 'config/' + extraConfigFile.getKey()) // wrap source file in closure to delay resolution to execution time - copyConfig.from({ extraConfigFile.getValue() }) { + copyConfig.from({ extraConfigFileValue }) { // this must be in a closure so it is only applied to the single file specified in from above into(configDir.toPath().relativize(destConfigFile.canonicalFile.parentFile.toPath()).toFile()) rename { destConfigFile.name } diff --git a/test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java similarity index 83% rename from test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java rename to buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java index 13163cee029..ed25d52739c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java +++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java @@ -25,14 +25,11 @@ import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.io.PathUtils; - /** * Checks that all tests in a directory are named according to our naming conventions. This is important because tests that do not follow * our conventions aren't run by gradle. This was once a glorious unit test but now that Elasticsearch is a multi-module project it must be @@ -46,11 +43,13 @@ import org.elasticsearch.common.io.PathUtils; * {@code --self-test} that is only run in the test:framework project. */ public class NamingConventionsCheck { - public static void main(String[] args) throws IOException, ClassNotFoundException { - NamingConventionsCheck check = new NamingConventionsCheck(); + public static void main(String[] args) throws IOException { + int i = 0; + NamingConventionsCheck check = new NamingConventionsCheck( + loadClassWithoutInitializing(args[i++]), + loadClassWithoutInitializing(args[i++])); boolean skipIntegTestsInDisguise = false; boolean selfTest = false; - int i = 0; while (true) { switch (args[i]) { case "--skip-integ-tests-in-disguise": @@ -69,7 +68,7 @@ public class NamingConventionsCheck { } break; } - check.check(PathUtils.get(args[i])); + check.check(Paths.get(args[i])); if (selfTest) { assertViolation("WrongName", check.missingSuffix); @@ -82,14 +81,12 @@ public class NamingConventionsCheck { } // Now we should have no violations - assertNoViolations("Not all subclasses of " + ESTestCase.class.getSimpleName() + assertNoViolations("Not all subclasses of " + check.testClass.getSimpleName() + " match the naming convention. Concrete classes must end with [Tests]", check.missingSuffix); assertNoViolations("Classes ending with [Tests] are abstract or interfaces", check.notRunnable); assertNoViolations("Found inner classes that are tests, which are excluded from the test runner", check.innerClasses); - String classesToSubclass = String.join(",", ESTestCase.class.getSimpleName(), ESTestCase.class.getSimpleName(), - ESTokenStreamTestCase.class.getSimpleName(), LuceneTestCase.class.getSimpleName()); - assertNoViolations("Pure Unit-Test found must subclass one of [" + classesToSubclass + "]", check.pureUnitTest); - assertNoViolations("Classes ending with [Tests] must subclass [" + classesToSubclass + "]", check.notImplementing); + assertNoViolations("Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]", check.pureUnitTest); + assertNoViolations("Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]", check.notImplementing); if (!skipIntegTestsInDisguise) { assertNoViolations("Subclasses of ESIntegTestCase should end with IT as they are integration tests", check.integTestsInDisguise); @@ -103,6 +100,14 @@ public class NamingConventionsCheck { private final Set> notRunnable = new HashSet<>(); private final Set> innerClasses = new HashSet<>(); + private final Class testClass; + private final Class integTestClass; + + public NamingConventionsCheck(Class testClass, Class integTestClass) { + this.testClass = testClass; + this.integTestClass = integTestClass; + } + public void check(Path rootPath) throws IOException { Files.walkFileTree(rootPath, new FileVisitor() { /** @@ -136,9 +141,9 @@ public class NamingConventionsCheck { String filename = file.getFileName().toString(); if (filename.endsWith(".class")) { String className = filename.substring(0, filename.length() - ".class".length()); - Class clazz = loadClass(className); + Class clazz = loadClassWithoutInitializing(packageName + className); if (clazz.getName().endsWith("Tests")) { - if (ESIntegTestCase.class.isAssignableFrom(clazz)) { + if (integTestClass.isAssignableFrom(clazz)) { integTestsInDisguise.add(clazz); } if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) { @@ -164,15 +169,7 @@ public class NamingConventionsCheck { } private boolean isTestCase(Class clazz) { - return LuceneTestCase.class.isAssignableFrom(clazz); - } - - private Class loadClass(String className) { - try { - return Thread.currentThread().getContextClassLoader().loadClass(packageName + className); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } + return testClass.isAssignableFrom(clazz); } @Override @@ -186,7 +183,6 @@ public class NamingConventionsCheck { * Fail the process if there are any violations in the set. Named to look like a junit assertion even though it isn't because it is * similar enough. */ - @SuppressForbidden(reason = "System.err/System.exit") private static void assertNoViolations(String message, Set> set) { if (false == set.isEmpty()) { System.err.println(message + ":"); @@ -201,10 +197,9 @@ public class NamingConventionsCheck { * Fail the process if we didn't detect a particular violation. Named to look like a junit assertion even though it isn't because it is * similar enough. */ - @SuppressForbidden(reason = "System.err/System.exit") - private static void assertViolation(String className, Set> set) throws ClassNotFoundException { - className = "org.elasticsearch.test.test.NamingConventionsCheckBadClasses$" + className; - if (false == set.remove(Class.forName(className))) { + private static void assertViolation(String className, Set> set) { + className = "org.elasticsearch.test.NamingConventionsCheckBadClasses$" + className; + if (false == set.remove(loadClassWithoutInitializing(className))) { System.err.println("Error in NamingConventionsCheck! Expected [" + className + "] to be a violation but wasn't."); System.exit(1); } @@ -213,9 +208,20 @@ public class NamingConventionsCheck { /** * Fail the process with the provided message. */ - @SuppressForbidden(reason = "System.err/System.exit") private static void fail(String reason) { System.err.println(reason); System.exit(1); } + + static Class loadClassWithoutInitializing(String name) { + try { + return Class.forName(name, + // Don't initialize the class to save time. Not needed for this test and this doesn't share a VM with any other tests. + false, + // Use our classloader rather than the bootstrap class loader. + NamingConventionsCheck.class.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } } diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 05ca294a9f0..d56bdeb537f 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -482,7 +482,6 @@ - diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt index 0e5ce884d9d..e31a7020282 100644 --- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt @@ -31,5 +31,3 @@ org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey() @defaultMessage Soon to be removed org.apache.lucene.document.FieldType#numericType() - -org.apache.lucene.document.InetAddressPoint#newPrefixQuery(java.lang.String, java.net.InetAddress, int) @LUCENE-7232 diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java similarity index 59% rename from test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java rename to buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java index 233e9fe5975..87240534880 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java +++ b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java @@ -17,9 +17,7 @@ * under the License. */ -package org.elasticsearch.test.test; - -import org.elasticsearch.test.ESTestCase; +package org.elasticsearch.test; import junit.framework.TestCase; @@ -30,21 +28,35 @@ public class NamingConventionsCheckBadClasses { public static final class NotImplementingTests { } - public static final class WrongName extends ESTestCase { + public static final class WrongName extends UnitTestCase { + /* + * Dummy test so the tests pass. We do this *and* skip the tests so anyone who jumps back to a branch without these tests can still + * compile without a failure. That is because clean doesn't actually clean these.... + */ + public void testDummy() {} } - public static abstract class DummyAbstractTests extends ESTestCase { + public static abstract class DummyAbstractTests extends UnitTestCase { } public interface DummyInterfaceTests { } - public static final class InnerTests extends ESTestCase { + public static final class InnerTests extends UnitTestCase { + public void testDummy() {} } - public static final class WrongNameTheSecond extends ESTestCase { + public static final class WrongNameTheSecond extends UnitTestCase { + public void testDummy() {} } public static final class PlainUnit extends TestCase { + public void testDummy() {} + } + + public abstract static class UnitTestCase extends TestCase { + } + + public abstract static class IntegTestCase extends UnitTestCase { } } diff --git a/buildSrc/version.properties b/buildSrc/version.properties index d9e3908df22..f8d8b5848c9 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ -elasticsearch = 5.0.0 -lucene = 6.0.1 +elasticsearch = 5.0.0-alpha4 +lucene = 6.1.0-snapshot-3a57bea # optional dependencies spatial4j = 0.6 @@ -17,3 +17,6 @@ httpclient = 4.5.2 httpcore = 4.4.4 commonslogging = 1.1.3 commonscodec = 1.10 + +# benchmark dependencies +jmh = 1.12 \ No newline at end of file diff --git a/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java b/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java deleted file mode 100644 index 580b875ce2c..00000000000 --- a/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.document; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; - -import org.apache.lucene.search.Query; -import org.apache.lucene.util.NumericUtils; -import org.elasticsearch.common.SuppressForbidden; - -/** - * Forked utility methods from Lucene's InetAddressPoint until LUCENE-7232 and - * LUCENE-7234 are released. - */ -// TODO: remove me when we upgrade to Lucene 6.1 -@SuppressForbidden(reason="uses InetAddress.getHostAddress") -public final class XInetAddressPoint { - - private XInetAddressPoint() {} - - /** The minimum value that an ip address can hold. */ - public static final InetAddress MIN_VALUE; - /** The maximum value that an ip address can hold. */ - public static final InetAddress MAX_VALUE; - static { - MIN_VALUE = InetAddressPoint.decode(new byte[InetAddressPoint.BYTES]); - byte[] maxValueBytes = new byte[InetAddressPoint.BYTES]; - Arrays.fill(maxValueBytes, (byte) 0xFF); - MAX_VALUE = InetAddressPoint.decode(maxValueBytes); - } - - /** - * Return the {@link InetAddress} that compares immediately greater than - * {@code address}. - * @throws ArithmeticException if the provided address is the - * {@link #MAX_VALUE maximum ip address} - */ - public static InetAddress nextUp(InetAddress address) { - if (address.equals(MAX_VALUE)) { - throw new ArithmeticException("Overflow: there is no greater InetAddress than " - + address.getHostAddress()); - } - byte[] delta = new byte[InetAddressPoint.BYTES]; - delta[InetAddressPoint.BYTES-1] = 1; - byte[] nextUpBytes = new byte[InetAddressPoint.BYTES]; - NumericUtils.add(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextUpBytes); - return InetAddressPoint.decode(nextUpBytes); - } - - /** - * Return the {@link InetAddress} that compares immediately less than - * {@code address}. - * @throws ArithmeticException if the provided address is the - * {@link #MIN_VALUE minimum ip address} - */ - public static InetAddress nextDown(InetAddress address) { - if (address.equals(MIN_VALUE)) { - throw new ArithmeticException("Underflow: there is no smaller InetAddress than " - + address.getHostAddress()); - } - byte[] delta = new byte[InetAddressPoint.BYTES]; - delta[InetAddressPoint.BYTES-1] = 1; - byte[] nextDownBytes = new byte[InetAddressPoint.BYTES]; - NumericUtils.subtract(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextDownBytes); - return InetAddressPoint.decode(nextDownBytes); - } - - /** - * Create a prefix query for matching a CIDR network range. - * - * @param field field name. must not be {@code null}. - * @param value any host address - * @param prefixLength the network prefix length for this address. This is also known as the subnet mask in the context of IPv4 - * addresses. - * @throws IllegalArgumentException if {@code field} is null, or prefixLength is invalid. - * @return a query matching documents with addresses contained within this network - */ - // TODO: remove me when we upgrade to Lucene 6.0.1 - public static Query newPrefixQuery(String field, InetAddress value, int prefixLength) { - if (value == null) { - throw new IllegalArgumentException("InetAddress must not be null"); - } - if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) { - throw new IllegalArgumentException("illegal prefixLength '" + prefixLength - + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges"); - } - // create the lower value by zeroing out the host portion, upper value by filling it with all ones. - byte lower[] = value.getAddress(); - byte upper[] = value.getAddress(); - for (int i = prefixLength; i < 8 * lower.length; i++) { - int m = 1 << (7 - (i & 7)); - lower[i >> 3] &= ~m; - upper[i >> 3] |= m; - } - try { - return InetAddressPoint.newRangeQuery(field, InetAddress.getByAddress(lower), InetAddress.getByAddress(upper)); - } catch (UnknownHostException e) { - throw new AssertionError(e); // values are coming from InetAddress - } - } -} diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 564f780b8ed..a4b94b007fd 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -283,7 +283,7 @@ public abstract class BlendedTermQuery extends Query { @Override public boolean equals(Object o) { if (this == o) return true; - if (!super.equals(o)) return false; + if (sameClassAs(o) == false) return false; BlendedTermQuery that = (BlendedTermQuery) o; return Arrays.equals(equalsTerms(), that.equalsTerms()); @@ -291,7 +291,7 @@ public abstract class BlendedTermQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), Arrays.hashCode(equalsTerms())); + return Objects.hash(classHash(), Arrays.hashCode(equalsTerms())); } public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) { diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index 86982bfc949..a8b7dc9299f 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -44,12 +44,12 @@ public final class MinDocQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), minDoc); + return Objects.hash(classHash(), minDoc); } @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } MinDocQuery that = (MinDocQuery) obj; diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java index a9327d785e1..6017803b63d 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java @@ -63,9 +63,6 @@ import org.elasticsearch.common.io.PathUtils; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -622,8 +619,12 @@ public long ramBytesUsed() { Set seenSurfaceForms = new HashSet<>(); int dedup = 0; - while (reader.read(scratch)) { - input.reset(scratch.bytes(), 0, scratch.length()); + while (true) { + BytesRef bytes = reader.next(); + if (bytes == null) { + break; + } + input.reset(bytes.bytes, bytes.offset, bytes.length); short analyzedLength = input.readShort(); analyzed.grow(analyzedLength+2); input.readBytes(analyzed.bytes(), 0, analyzedLength); @@ -631,13 +632,13 @@ public long ramBytesUsed() { long cost = input.readInt(); - surface.bytes = scratch.bytes(); + surface.bytes = bytes.bytes; if (hasPayloads) { surface.length = input.readShort(); surface.offset = input.getPosition(); } else { surface.offset = input.getPosition(); - surface.length = scratch.length() - surface.offset; + surface.length = bytes.length - surface.offset; } if (previousAnalyzed == null) { @@ -679,11 +680,11 @@ public long ramBytesUsed() { builder.add(scratchInts.get(), outputs.newPair(cost, BytesRef.deepCopyOf(surface))); } else { int payloadOffset = input.getPosition() + surface.length; - int payloadLength = scratch.length() - payloadOffset; + int payloadLength = bytes.length - payloadOffset; BytesRef br = new BytesRef(surface.length + 1 + payloadLength); System.arraycopy(surface.bytes, surface.offset, br.bytes, 0, surface.length); br.bytes[surface.length] = (byte) payloadSep; - System.arraycopy(scratch.bytes(), payloadOffset, br.bytes, surface.length+1, payloadLength); + System.arraycopy(bytes.bytes, payloadOffset, br.bytes, surface.length+1, payloadLength); br.length = br.bytes.length; builder.add(scratchInts.get(), outputs.newPair(cost, br)); } diff --git a/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java b/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java index d38de2e3bc1..d408fdef033 100644 --- a/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java +++ b/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java @@ -32,7 +32,7 @@ public class ResourceNotFoundException extends ElasticsearchException { super(msg, args); } - protected ResourceNotFoundException(String msg, Throwable cause, Object... args) { + public ResourceNotFoundException(String msg, Throwable cause, Object... args) { super(msg, cause, args); } diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 5bdbf76265d..e77b0f450dd 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -76,9 +76,9 @@ public class Version { public static final Version V_5_0_0_alpha2 = new Version(V_5_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); public static final int V_5_0_0_alpha3_ID = 5000003; public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); - public static final int V_5_0_0_ID = 5000099; - public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_1); - public static final Version CURRENT = V_5_0_0; + public static final int V_5_0_0_alpha4_ID = 5000004; + public static final Version V_5_0_0_alpha4 = new Version(V_5_0_0_alpha4_ID, org.apache.lucene.util.Version.LUCENE_6_1_0); + public static final Version CURRENT = V_5_0_0_alpha4; static { assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to [" @@ -91,8 +91,8 @@ public class Version { public static Version fromId(int id) { switch (id) { - case V_5_0_0_ID: - return V_5_0_0; + case V_5_0_0_alpha4_ID: + return V_5_0_0_alpha4; case V_5_0_0_alpha3_ID: return V_5_0_0_alpha3; case V_5_0_0_alpha2_ID: diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index 7e975b922d4..c7e6d795afc 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -32,6 +32,8 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction; +import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryAction; @@ -141,7 +143,7 @@ import org.elasticsearch.action.delete.TransportDeleteAction; import org.elasticsearch.action.explain.ExplainAction; import org.elasticsearch.action.explain.TransportExplainAction; import org.elasticsearch.action.fieldstats.FieldStatsAction; -import org.elasticsearch.action.fieldstats.TransportFieldStatsTransportAction; +import org.elasticsearch.action.fieldstats.TransportFieldStatsAction; import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.MultiGetAction; import org.elasticsearch.action.get.TransportGetAction; @@ -264,6 +266,7 @@ public class ActionModule extends AbstractModule { registerAction(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class); registerAction(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class); registerAction(ListTasksAction.INSTANCE, TransportListTasksAction.class); + registerAction(GetTaskAction.INSTANCE, TransportGetTaskAction.class); registerAction(CancelTasksAction.INSTANCE, TransportCancelTasksAction.class); registerAction(ClusterAllocationExplainAction.INSTANCE, TransportClusterAllocationExplainAction.class); @@ -341,7 +344,7 @@ public class ActionModule extends AbstractModule { registerAction(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class); registerAction(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class); - registerAction(FieldStatsAction.INSTANCE, TransportFieldStatsTransportAction.class); + registerAction(FieldStatsAction.INSTANCE, TransportFieldStatsAction.class); registerAction(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class); registerAction(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class); diff --git a/core/src/main/java/org/elasticsearch/action/ActionRequest.java b/core/src/main/java/org/elasticsearch/action/ActionRequest.java index dac3b4f2a21..bc052895a6f 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ActionRequest.java @@ -39,6 +39,9 @@ public abstract class ActionRequest> exte public abstract ActionRequestValidationException validate(); + /** + * Should this task persist its result after it has finished? + */ public boolean getShouldPersistResult() { return false; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 59b426d8c31..27970f332fc 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -33,8 +33,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; - /** * */ @@ -160,7 +158,7 @@ public class ClusterHealthRequest extends MasterNodeReadRequest { + + public static final GetTaskAction INSTANCE = new GetTaskAction(); + public static final String NAME = "cluster:monitor/task/get"; + + private GetTaskAction() { + super(NAME); + } + + @Override + public GetTaskResponse newResponse() { + return new GetTaskResponse(); + } + + @Override + public GetTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new GetTaskRequestBuilder(client, this); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java new file mode 100644 index 00000000000..efbc9679e71 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.tasks.get; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.TaskId; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * A request to get node tasks + */ +public class GetTaskRequest extends ActionRequest { + private TaskId taskId = TaskId.EMPTY_TASK_ID; + private boolean waitForCompletion = false; + private TimeValue timeout = null; + + /** + * Get the TaskId to look up. + */ + public TaskId getTaskId() { + return taskId; + } + + /** + * Set the TaskId to look up. Required. + */ + public GetTaskRequest setTaskId(TaskId taskId) { + this.taskId = taskId; + return this; + } + + /** + * Should this request wait for all found tasks to complete? + */ + public boolean getWaitForCompletion() { + return waitForCompletion; + } + + /** + * Should this request wait for all found tasks to complete? + */ + public GetTaskRequest setWaitForCompletion(boolean waitForCompletion) { + this.waitForCompletion = waitForCompletion; + return this; + } + + /** + * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2. + */ + public TimeValue getTimeout() { + return timeout; + } + + /** + * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2. + */ + public GetTaskRequest setTimeout(TimeValue timeout) { + this.timeout = timeout; + return this; + } + + GetTaskRequest nodeRequest(String thisNodeId, long thisTaskId) { + GetTaskRequest copy = new GetTaskRequest(); + copy.setParentTask(thisNodeId, thisTaskId); + copy.setTaskId(taskId); + copy.setTimeout(timeout); + copy.setWaitForCompletion(waitForCompletion); + return copy; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (false == getTaskId().isSet()) { + validationException = addValidationError("task id is required", validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = TaskId.readFromStream(in); + timeout = in.readOptionalWriteable(TimeValue::new); + waitForCompletion = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + taskId.writeTo(out); + out.writeOptionalWriteable(timeout); + out.writeBoolean(waitForCompletion); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java new file mode 100644 index 00000000000..e1042df2ac3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.tasks.get; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.TaskId; + +/** + * Builder for the request to retrieve the list of tasks running on the specified nodes + */ +public class GetTaskRequestBuilder extends ActionRequestBuilder { + public GetTaskRequestBuilder(ElasticsearchClient client, GetTaskAction action) { + super(client, action, new GetTaskRequest()); + } + + /** + * Set the TaskId to look up. Required. + */ + public final GetTaskRequestBuilder setTaskId(TaskId taskId) { + request.setTaskId(taskId); + return this; + } + + /** + * Should this request wait for all found tasks to complete? + */ + public final GetTaskRequestBuilder setWaitForCompletion(boolean waitForCompletion) { + request.setWaitForCompletion(waitForCompletion); + return this; + } + + /** + * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2. + */ + public final GetTaskRequestBuilder setTimeout(TimeValue timeout) { + request.setTimeout(timeout); + return this; + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java new file mode 100644 index 00000000000..afb03a7c9dc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.tasks.get; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.PersistedTaskInfo; + +import java.io.IOException; + +import static java.util.Objects.requireNonNull; + +/** + * Returns the list of tasks currently running on the nodes + */ +public class GetTaskResponse extends ActionResponse implements ToXContent { + private PersistedTaskInfo task; + + public GetTaskResponse() { + } + + public GetTaskResponse(PersistedTaskInfo task) { + this.task = requireNonNull(task, "task is required"); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + task = in.readOptionalWriteable(PersistedTaskInfo::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(task); + } + + /** + * Get the actual result of the fetch. + */ + public PersistedTaskInfo getTask() { + return task; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return task.innerToXContent(builder, params); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java new file mode 100644 index 00000000000..769676bcd6d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java @@ -0,0 +1,216 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.node.tasks.get; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.PersistedTaskInfo; +import org.elasticsearch.tasks.TaskPersistenceService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.BaseTransportResponseHandler; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; + +import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForCompletionTimeout; +import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForTaskCompletion; + +/** + * Action to get a single task. If the task isn't running then it'll try to request the status from request index. + * + * The general flow is: + *
    + *
  • If this isn't being executed on the node to which the requested TaskId belongs then move to that node. + *
  • Look up the task and return it if it exists + *
  • If it doesn't then look up the task from the results index + *
+ */ +public class TransportGetTaskAction extends HandledTransportAction { + private final ClusterService clusterService; + private final TransportService transportService; + private final Client client; + + @Inject + public TransportGetTaskAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client) { + super(settings, GetTaskAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetTaskRequest::new); + this.clusterService = clusterService; + this.transportService = transportService; + this.client = client; + } + + @Override + protected void doExecute(GetTaskRequest request, ActionListener listener) { + throw new UnsupportedOperationException("Task is required"); + } + + @Override + protected void doExecute(Task thisTask, GetTaskRequest request, ActionListener listener) { + if (clusterService.localNode().getId().equals(request.getTaskId().getNodeId())) { + getRunningTaskFromNode(thisTask, request, listener); + } else { + runOnNodeWithTaskIfPossible(thisTask, request, listener); + } + } + + /** + * Executed on the coordinating node to forward execution of the remaining work to the node that matches that requested + * {@link TaskId#getNodeId()}. If the node isn't in the cluster then this will just proceed to + * {@link #getFinishedTaskFromIndex(Task, GetTaskRequest, ActionListener)} on this node. + */ + private void runOnNodeWithTaskIfPossible(Task thisTask, GetTaskRequest request, ActionListener listener) { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + if (request.getTimeout() != null) { + builder.withTimeout(request.getTimeout()); + } + builder.withCompress(false); + DiscoveryNode node = clusterService.state().nodes().get(request.getTaskId().getNodeId()); + if (node == null) { + // Node is no longer part of the cluster! Try and look the task up from the results index. + getFinishedTaskFromIndex(thisTask, request, listener); + return; + } + GetTaskRequest nodeRequest = request.nodeRequest(clusterService.localNode().getId(), thisTask.getId()); + taskManager.registerChildTask(thisTask, node.getId()); + transportService.sendRequest(node, GetTaskAction.NAME, nodeRequest, builder.build(), + new BaseTransportResponseHandler() { + @Override + public GetTaskResponse newInstance() { + return new GetTaskResponse(); + } + + @Override + public void handleResponse(GetTaskResponse response) { + listener.onResponse(response); + } + + @Override + public void handleException(TransportException exp) { + listener.onFailure(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.SAME; + } + }); + } + + /** + * Executed on the node that should be running the task to find and return the running task. Falls back to + * {@link #getFinishedTaskFromIndex(Task, GetTaskRequest, ActionListener)} if the task isn't still running. + */ + void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListener listener) { + Task runningTask = taskManager.getTask(request.getTaskId().getId()); + if (runningTask == null) { + getFinishedTaskFromIndex(thisTask, request, listener); + } else { + if (request.getWaitForCompletion()) { + // Shift to the generic thread pool and let it wait for the task to complete so we don't block any important threads. + threadPool.generic().execute(new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + waitForTaskCompletion(taskManager, runningTask, waitForCompletionTimeout(request.getTimeout())); + // TODO look up the task's result from the .tasks index now that it is done + listener.onResponse( + new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true)))); + } + + @Override + public void onFailure(Throwable t) { + listener.onFailure(t); + } + }); + } else { + listener.onResponse(new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true)))); + } + } + } + + /** + * Send a {@link GetRequest} to the results index looking for the results of the task. It'll only be found only if the task's result was + * persisted. Called on the node that once had the task if that node is part of the cluster or on the coordinating node if the node + * wasn't part of the cluster. + */ + void getFinishedTaskFromIndex(Task thisTask, GetTaskRequest request, ActionListener listener) { + GetRequest get = new GetRequest(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE, + request.getTaskId().toString()); + get.setParentTask(clusterService.localNode().getId(), thisTask.getId()); + client.get(get, new ActionListener() { + @Override + public void onResponse(GetResponse getResponse) { + try { + onGetFinishedTaskFromIndex(getResponse, listener); + } catch (Throwable e) { + listener.onFailure(e); + } + } + + @Override + public void onFailure(Throwable e) { + if (ExceptionsHelper.unwrap(e, IndexNotFoundException.class) != null) { + // We haven't yet created the index for the task results so it can't be found. + listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", e, request.getTaskId())); + } else { + listener.onFailure(e); + } + } + }); + } + + /** + * Called with the {@linkplain GetResponse} from loading the task from the results index. Called on the node that once had the task if + * that node is part of the cluster or on the coordinating node if the node wasn't part of the cluster. + */ + void onGetFinishedTaskFromIndex(GetResponse response, ActionListener listener) throws IOException { + if (false == response.isExists()) { + listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", response.getId())); + } + if (response.isSourceEmpty()) { + listener.onFailure(new ElasticsearchException("Stored task status for [{}] didn't contain any source!", response.getId())); + return; + } + try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) { + PersistedTaskInfo result = PersistedTaskInfo.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT); + listener.onResponse(new GetTaskResponse(result)); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java index bad4001e211..6ab0bafb2fb 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java @@ -23,21 +23,21 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.support.tasks.BaseTasksResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; /** @@ -47,10 +47,12 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent { private List tasks; - private Map> nodes; + private Map> perNodeTasks; private List groups; + private DiscoveryNodes discoveryNodes; + public ListTasksResponse() { } @@ -75,28 +77,11 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent { /** * Returns the list of tasks by node */ - public Map> getPerNodeTasks() { - if (nodes != null) { - return nodes; + public Map> getPerNodeTasks() { + if (perNodeTasks == null) { + perNodeTasks = tasks.stream().collect(Collectors.groupingBy(t -> t.getTaskId().getNodeId())); } - Map> nodeTasks = new HashMap<>(); - - Set nodes = new HashSet<>(); - for (TaskInfo shard : tasks) { - nodes.add(shard.getNode()); - } - - for (DiscoveryNode node : nodes) { - List tasks = new ArrayList<>(); - for (TaskInfo taskInfo : this.tasks) { - if (taskInfo.getNode().equals(node)) { - tasks.add(taskInfo); - } - } - nodeTasks.put(node, tasks); - } - this.nodes = nodeTasks; - return nodeTasks; + return perNodeTasks; } public List getTaskGroups() { @@ -138,6 +123,14 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent { return tasks; } + /** + * Set a reference to the {@linkplain DiscoveryNodes}. Used for calling {@link #toXContent(XContentBuilder, ToXContent.Params)} with + * {@code group_by=nodes}. + */ + public void setDiscoveryNodes(DiscoveryNodes discoveryNodes) { + this.discoveryNodes = discoveryNodes; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (getTaskFailures() != null && getTaskFailures().size() > 0) { @@ -161,33 +154,38 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent { } String groupBy = params.param("group_by", "nodes"); if ("nodes".equals(groupBy)) { + if (discoveryNodes == null) { + throw new IllegalStateException("discoveryNodes must be set before calling toXContent with group_by=nodes"); + } builder.startObject("nodes"); - for (Map.Entry> entry : getPerNodeTasks().entrySet()) { - DiscoveryNode node = entry.getKey(); - builder.startObject(node.getId()); - builder.field("name", node.getName()); - builder.field("transport_address", node.getAddress().toString()); - builder.field("host", node.getHostName()); - builder.field("ip", node.getAddress()); + for (Map.Entry> entry : getPerNodeTasks().entrySet()) { + DiscoveryNode node = discoveryNodes.get(entry.getKey()); + builder.startObject(entry.getKey()); + if (node != null) { + // If the node is no longer part of the cluster, oh well, we'll just skip it's useful information. + builder.field("name", node.getName()); + builder.field("transport_address", node.getAddress().toString()); + builder.field("host", node.getHostName()); + builder.field("ip", node.getAddress()); - builder.startArray("roles"); - for (DiscoveryNode.Role role : node.getRoles()) { - builder.value(role.getRoleName()); - } - builder.endArray(); - - if (!node.getAttributes().isEmpty()) { - builder.startObject("attributes"); - for (Map.Entry attrEntry : node.getAttributes().entrySet()) { - builder.field(attrEntry.getKey(), attrEntry.getValue()); + builder.startArray("roles"); + for (DiscoveryNode.Role role : node.getRoles()) { + builder.value(role.getRoleName()); + } + builder.endArray(); + + if (!node.getAttributes().isEmpty()) { + builder.startObject("attributes"); + for (Map.Entry attrEntry : node.getAttributes().entrySet()) { + builder.field(attrEntry.getKey(), attrEntry.getValue()); + } + builder.endObject(); } - builder.endObject(); } builder.startObject("tasks"); for(TaskInfo task : entry.getValue()) { - builder.startObject(task.getTaskId().toString()); + builder.field(task.getTaskId().toString()); task.toXContent(builder, params); - builder.endObject(); } builder.endObject(); builder.endObject(); @@ -196,9 +194,8 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent { } else if ("parents".equals(groupBy)) { builder.startObject("tasks"); for (TaskGroup group : getTaskGroups()) { - builder.startObject(group.getTaskInfo().getTaskId().toString()); + builder.field(group.getTaskInfo().getTaskId().toString()); group.toXContent(builder, params); - builder.endObject(); } builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java index aa9bfd6b720..b254137163d 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java @@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks.list; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; import java.util.ArrayList; @@ -79,16 +80,15 @@ public class TaskGroup implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - task.toXContent(builder, params); + builder.startObject(); + task.innerToXContent(builder, params); if (childTasks.isEmpty() == false) { builder.startArray("children"); for (TaskGroup taskGroup : childTasks) { - builder.startObject(); taskGroup.toXContent(builder, params); - builder.endObject(); } builder.endArray(); } - return builder; + return builder.endObject(); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java index b05049f6776..7d6d122bbb7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java @@ -33,6 +33,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -47,6 +49,26 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; * */ public class TransportListTasksAction extends TransportTasksAction { + public static void waitForTaskCompletion(TaskManager taskManager, Task task, long untilInNanos) { + while (System.nanoTime() - untilInNanos < 0) { + if (taskManager.getTask(task.getId()) == null) { + return; + } + try { + Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis()); + } catch (InterruptedException e) { + throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task); + } + } + throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task); + } + public static long waitForCompletionTimeout(TimeValue timeout) { + if (timeout == null) { + timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT; + } + return System.nanoTime() + timeout.nanos(); + } + private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100); private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30); @@ -75,35 +97,18 @@ public class TransportListTasksAction extends TransportTasksAction operation) { - if (false == request.getWaitForCompletion()) { - super.processTasks(request, operation); - return; - } - // If we should wait for completion then we have to intercept every found task and wait for it to leave the manager. - TimeValue timeout = request.getTimeout(); - if (timeout == null) { - timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT; - } - long timeoutTime = System.nanoTime() + timeout.nanos(); - super.processTasks(request, operation.andThen((Task t) -> { - while (System.nanoTime() - timeoutTime < 0) { - Task task = taskManager.getTask(t.getId()); - if (task == null) { - return; - } + if (request.getWaitForCompletion()) { + long timeoutNanos = waitForCompletionTimeout(request.getTimeout()); + operation = operation.andThen(task -> { if (task.getAction().startsWith(ListTasksAction.NAME)) { // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting - // for itself of one of its child tasks + // for itself or one of its child tasks return; } - try { - Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis()); - } catch (InterruptedException e) { - throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, t); - } - } - throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", t); - })); + waitForTaskCompletion(taskManager, task, timeoutNanos); + }); + } + super.processTasks(request, operation); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 3d287eee9d1..a7cebca0aa6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -141,7 +141,7 @@ public class SnapshotIndexShardStatus extends BroadcastShardResponse implements @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Integer.toString(getShardId())); + builder.startObject(Integer.toString(getShardId().getId())); builder.field(Fields.STAGE, getStage()); stats.toXContent(builder, params); if (getNodeId() != null) { diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java index 16c361d5ca6..8f2f7ed5f50 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java @@ -49,7 +49,7 @@ public class SnapshotIndexStatus implements Iterable, Map indexShards = new HashMap<>(); stats = new SnapshotStats(); for (SnapshotIndexShardStatus shard : shards) { - indexShards.put(shard.getShardId(), shard); + indexShards.put(shard.getShardId().getId(), shard); stats.add(shard.getStats()); } shardsStats = new SnapshotShardsStats(shards); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 85d7147ada0..e0572344656 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -544,7 +544,7 @@ public class BulkRequest extends ActionRequest implements Composite } } refreshPolicy = RefreshPolicy.readFrom(in); - timeout = TimeValue.readTimeValue(in); + timeout = new TimeValue(in); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 4cbebd0739a..c6b046a4758 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -304,7 +304,7 @@ public class TransportBulkAction extends HandledTransportAction list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); @@ -314,7 +314,7 @@ public class TransportBulkAction extends HandledTransportAction list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); @@ -324,7 +324,7 @@ public class TransportBulkAction extends HandledTransportAction list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index 88e7c66c1b1..1084cf6c237 100644 --- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -152,7 +152,7 @@ public class TransportExplainAction extends TransportSingleShardAction { private String[] fields = Strings.EMPTY_ARRAY; private String level = DEFAULT_LEVEL; private IndexConstraint[] indexConstraints = new IndexConstraint[0]; + private boolean useCache = true; public String[] getFields() { return fields; @@ -56,6 +57,14 @@ public class FieldStatsRequest extends BroadcastRequest { this.fields = fields; } + public void setUseCache(boolean useCache) { + this.useCache = useCache; + } + + public boolean shouldUseCache() { + return useCache; + } + public IndexConstraint[] getIndexConstraints() { return indexConstraints; } @@ -184,6 +193,7 @@ public class FieldStatsRequest extends BroadcastRequest { indexConstraints[i] = new IndexConstraint(in); } level = in.readString(); + useCache = in.readBoolean(); } @Override @@ -201,6 +211,7 @@ public class FieldStatsRequest extends BroadcastRequest { } } out.writeString(level); + out.writeBoolean(useCache); } } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java index c0c4d78de9b..1a3a8070e46 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java @@ -45,4 +45,9 @@ public class FieldStatsRequestBuilder extends request().level(level); return this; } + + public FieldStatsRequestBuilder setUseCache(boolean useCache) { + request().setUseCache(useCache); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java index 6705bd0e0b5..85a0d469541 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java @@ -34,6 +34,7 @@ import java.util.Set; public class FieldStatsShardRequest extends BroadcastShardRequest { private String[] fields; + private boolean useCache; public FieldStatsShardRequest() { } @@ -46,22 +47,29 @@ public class FieldStatsShardRequest extends BroadcastShardRequest { fields.add(indexConstraint.getField()); } this.fields = fields.toArray(new String[fields.size()]); + useCache = request.shouldUseCache(); } public String[] getFields() { return fields; } + public boolean shouldUseCache() { + return useCache; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); fields = in.readStringArray(); + useCache = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArrayNullable(fields); + out.writeBoolean(useCache); } } diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java index a3043d3ae35..7cc298729f0 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java @@ -46,7 +46,6 @@ public class FieldStatsShardResponse extends BroadcastShardResponse { return fieldStats; } - @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java similarity index 90% rename from core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java rename to core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java index 4d2d1db161d..f70e5dda114 100644 --- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java +++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; @@ -45,27 +44,23 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.io.IOException; - -import java.util.Map; -import java.util.HashMap; -import java.util.List; import java.util.ArrayList; -import java.util.Iterator; -import java.util.Set; -import java.util.HashSet; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReferenceArray; -public class TransportFieldStatsTransportAction extends +public class TransportFieldStatsAction extends TransportBroadcastAction { private final IndicesService indicesService; @Inject - public TransportFieldStatsTransportAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, + public TransportFieldStatsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService) { @@ -192,29 +187,20 @@ public class TransportFieldStatsTransportAction extends ShardId shardId = request.shardId(); Map> fieldStats = new HashMap<>(); IndexService indexServices = indicesService.indexServiceSafe(shardId.getIndex()); - MapperService mapperService = indexServices.mapperService(); IndexShard shard = indexServices.getShard(shardId.id()); try (Engine.Searcher searcher = shard.acquireSearcher("fieldstats")) { + // Resolve patterns and deduplicate + Set fieldNames = new HashSet<>(); for (String field : request.getFields()) { - Collection matchFields; - if (Regex.isSimpleMatchPattern(field)) { - matchFields = mapperService.simpleMatchToIndexNames(field); - } else { - matchFields = Collections.singleton(field); - } - for (String matchField : matchFields) { - MappedFieldType fieldType = mapperService.fullName(matchField); - if (fieldType == null) { - // ignore. - continue; - } - FieldStats stats = fieldType.stats(searcher.reader()); - if (stats != null) { - fieldStats.put(matchField, stats); - } + fieldNames.addAll(shard.mapperService().simpleMatchToIndexNames(field)); + } + for (String field : fieldNames) { + FieldStats stats = indicesService.getFieldStats(shard, searcher, field, request.shouldUseCache()); + if (stats != null) { + fieldStats.put(field, stats); } } - } catch (IOException e) { + } catch (Exception e) { throw ExceptionsHelper.convertToElastic(e); } return new FieldStatsShardResponse(shardId, fieldStats); diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 528546d5b31..240035aee2a 100644 --- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -62,7 +62,7 @@ public class TransportGetAction extends TransportSingleShardAction implement routing = in.readOptionalString(); parent = in.readOptionalString(); timestamp = in.readOptionalString(); - ttl = in.readBoolean() ? TimeValue.readTimeValue(in) : null; + ttl = in.readOptionalWriteable(TimeValue::new); source = in.readBytesReference(); - opType = OpType.fromId(in.readByte()); version = in.readLong(); versionType = VersionType.fromValue(in.readByte()); @@ -650,12 +649,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement out.writeOptionalString(routing); out.writeOptionalString(parent); out.writeOptionalString(timestamp); - if (ttl == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - ttl.writeTo(out); - } + out.writeOptionalWriteable(ttl); out.writeBytesReference(source); out.writeByte(opType.id()); out.writeLong(version); diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index a3236e9653f..08a1ec5b3de 100644 --- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -38,6 +38,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; */ public class MultiSearchRequest extends ActionRequest implements CompositeIndicesRequest { + private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); @@ -60,6 +61,25 @@ public class MultiSearchRequest extends ActionRequest implem return this; } + /** + * Returns the amount of search requests specified in this multi search requests are allowed to be ran concurrently. + */ + public int maxConcurrentSearchRequests() { + return maxConcurrentSearchRequests; + } + + /** + * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently. + */ + public MultiSearchRequest maxConcurrentSearchRequests(int maxConcurrentSearchRequests) { + if (maxConcurrentSearchRequests < 1) { + throw new IllegalArgumentException("maxConcurrentSearchRequests must be positive"); + } + + this.maxConcurrentSearchRequests = maxConcurrentSearchRequests; + return this; + } + public List requests() { return this.requests; } @@ -100,6 +120,7 @@ public class MultiSearchRequest extends ActionRequest implem @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); + maxConcurrentSearchRequests = in.readVInt(); int size = in.readVInt(); for (int i = 0; i < size; i++) { SearchRequest request = new SearchRequest(); @@ -111,6 +132,7 @@ public class MultiSearchRequest extends ActionRequest implem @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); + out.writeVInt(maxConcurrentSearchRequests); out.writeVInt(requests.size()); for (SearchRequest request : requests) { request.writeTo(out); diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java index a0d1e4fb5c5..6cebb73fb4f 100644 --- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java @@ -71,4 +71,12 @@ public class MultiSearchRequestBuilder extends ActionRequestBuilder> getProfileResults() { + public @Nullable Map getProfileResults() { return internalResponse.profile(); } diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java index c0428cd531f..bd1751856b2 100644 --- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -29,57 +30,118 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; -/** - */ public class TransportMultiSearchAction extends HandledTransportAction { + private final int availableProcessors; private final ClusterService clusterService; - private final TransportSearchAction searchAction; + private final TransportAction searchAction; @Inject public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService, - ClusterService clusterService, TransportSearchAction searchAction, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + ClusterService clusterService, TransportSearchAction searchAction, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { super(settings, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new); this.clusterService = clusterService; this.searchAction = searchAction; + this.availableProcessors = EsExecutors.boundedNumberOfProcessors(settings); + } + + // For testing only: + TransportMultiSearchAction(ThreadPool threadPool, ActionFilters actionFilters, TransportService transportService, + ClusterService clusterService, TransportAction searchAction, + IndexNameExpressionResolver indexNameExpressionResolver, int availableProcessors) { + super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new); + this.clusterService = clusterService; + this.searchAction = searchAction; + this.availableProcessors = availableProcessors; } @Override - protected void doExecute(final MultiSearchRequest request, final ActionListener listener) { + protected void doExecute(MultiSearchRequest request, ActionListener listener) { ClusterState clusterState = clusterService.state(); clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); - final AtomicArray responses = new AtomicArray<>(request.requests().size()); - final AtomicInteger counter = new AtomicInteger(responses.length()); - for (int i = 0; i < responses.length(); i++) { - final int index = i; - searchAction.execute(request.requests().get(i), new ActionListener() { - @Override - public void onResponse(SearchResponse searchResponse) { - responses.set(index, new MultiSearchResponse.Item(searchResponse, null)); - if (counter.decrementAndGet() == 0) { - finishHim(); - } - } + int maxConcurrentSearches = request.maxConcurrentSearchRequests(); + if (maxConcurrentSearches == 0) { + maxConcurrentSearches = defaultMaxConcurrentSearches(availableProcessors, clusterState); + } - @Override - public void onFailure(Throwable e) { - responses.set(index, new MultiSearchResponse.Item(null, e)); - if (counter.decrementAndGet() == 0) { - finishHim(); - } - } + Queue searchRequestSlots = new ConcurrentLinkedQueue<>(); + for (int i = 0; i < request.requests().size(); i++) { + SearchRequest searchRequest = request.requests().get(i); + searchRequestSlots.add(new SearchRequestSlot(searchRequest, i)); + } - private void finishHim() { + int numRequests = request.requests().size(); + final AtomicArray responses = new AtomicArray<>(numRequests); + final AtomicInteger responseCounter = new AtomicInteger(numRequests); + int numConcurrentSearches = Math.min(numRequests, maxConcurrentSearches); + for (int i = 0; i < numConcurrentSearches; i++) { + executeSearch(searchRequestSlots, responses, responseCounter, listener); + } + } + + /* + * This is not perfect and makes a big assumption, that all nodes have the same thread pool size / have the number + * of processors and that shard of the indices the search requests go to are more or less evenly distributed across + * all nodes in the cluster. But I think it is a good enough default for most cases, if not then the default should be + * overwritten in the request itself. + */ + static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState state) { + int numDateNodes = state.getNodes().getDataNodes().size(); + // availableProcessors will never be larger than 32, so max defaultMaxConcurrentSearches will never be larger than 49, + // but we don't know about about other search requests that are being executed so lets cap at 10 per node + int defaultSearchThreadPoolSize = Math.min(ThreadPool.searchThreadPoolSize(availableProcessors), 10); + return Math.max(1, numDateNodes * defaultSearchThreadPoolSize); + } + + void executeSearch(Queue requests, AtomicArray responses, + AtomicInteger responseCounter, ActionListener listener) { + SearchRequestSlot request = requests.poll(); + if (request == null) { + // Ok... so there're no more requests then this is ok, we're then waiting for running requests to complete + return; + } + searchAction.execute(request.request, new ActionListener() { + @Override + public void onResponse(SearchResponse searchResponse) { + responses.set(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null)); + handleResponse(); + } + + @Override + public void onFailure(Throwable e) { + responses.set(request.responseSlot, new MultiSearchResponse.Item(null, e)); + handleResponse(); + } + + private void handleResponse() { + if (responseCounter.decrementAndGet() == 0) { listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()]))); + } else { + executeSearch(requests, responses, responseCounter, listener); } - }); + } + }); + } + + final static class SearchRequestSlot { + + final SearchRequest request; + final int responseSlot; + + SearchRequestSlot(SearchRequest request, int responseSlot) { + this.request = request; + this.responseSlot = responseSlot; } } } diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java index ad79285051a..398a8d6c905 100644 --- a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java +++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java @@ -45,8 +45,8 @@ public abstract class BroadcastShardResponse extends TransportResponse { return this.shardId.getIndexName(); } - public int getShardId() { - return this.shardId.id(); + public ShardId getShardId() { + return this.shardId; } @Override diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java index 9e45bccc547..e3f32543bf2 100644 --- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; /** @@ -75,7 +74,7 @@ public abstract class AcknowledgedRequest public void readFrom(StreamInput in) throws IOException { super.readFrom(in); nodesIds = in.readStringArray(); - if (in.readBoolean()) { - timeout = TimeValue.readTimeValue(in); - } + timeout = in.readOptionalWriteable(TimeValue::new); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArrayNullable(nodesIds); - if (timeout == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - timeout.writeTo(out); - } + out.writeOptionalWriteable(timeout); } } diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java index 44c420598b5..c444fd2cf39 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java @@ -181,7 +181,7 @@ public abstract class ReplicationRequest> extends parentTaskId = TaskId.readFromStream(in); nodesIds = in.readStringArray(); actions = in.readStringArray(); - if (in.readBoolean()) { - timeout = TimeValue.readTimeValue(in); - } + timeout = in.readOptionalWriteable(TimeValue::new); } @Override @@ -156,7 +154,7 @@ public class BaseTasksRequest> extends parentTaskId.writeTo(out); out.writeStringArrayNullable(nodesIds); out.writeStringArrayNullable(actions); - out.writeOptionalStreamable(timeout); + out.writeOptionalWriteable(timeout); } public boolean match(Task task) { diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java index 78a2de20a89..a3528cb75c4 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.TaskId; /** * Builder for task-based requests @@ -36,6 +37,15 @@ public class TasksRequestBuilder< super(client, action, request); } + /** + * Set the task to lookup. + */ + @SuppressWarnings("unchecked") + public final RequestBuilder setTaskId(TaskId taskId) { + request.setTaskId(taskId); + return (RequestBuilder) this; + } + @SuppressWarnings("unchecked") public final RequestBuilder setNodesIds(String... nodesIds) { request.setNodesIds(nodesIds); diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java index 47eae68f2e8..55885b03696 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.termvectors; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; @@ -56,7 +55,7 @@ public class TransportTermVectorsAction extends TransportSingleShardAction { } private JavaVersion(List version) { + if (version.size() >= 2 + && version.get(0).intValue() == 1 + && version.get(1).intValue() == 8) { + // for Java 8 there is ambiguity since both 1.8 and 8 are supported, + // so we rewrite the former to the latter + version = new ArrayList<>(version.subList(1, version.size())); + } this.version = Collections.unmodifiableList(version); } @@ -75,6 +82,19 @@ public class JavaVersion implements Comparable { return 0; } + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != getClass()) { + return false; + } + return compareTo((JavaVersion) o) == 0; + } + + @Override + public int hashCode() { + return version.hashCode(); + } + @Override public String toString() { return version.stream().map(v -> Integer.toString(v)).collect(Collectors.joining(".")); diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java index 37886239195..953ca578579 100644 --- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java @@ -39,6 +39,9 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -112,6 +115,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.tasks.TaskId; /** * Administrative actions/operations against indices. @@ -303,6 +307,34 @@ public interface ClusterAdminClient extends ElasticsearchClient { */ ListTasksRequestBuilder prepareListTasks(String... nodesIds); + /** + * Get a task. + * + * @param request the request + * @return the result future + * @see org.elasticsearch.client.Requests#getTaskRequest() + */ + ActionFuture getTask(GetTaskRequest request); + + /** + * Get a task. + * + * @param request the request + * @param listener A listener to be notified with the result + * @see org.elasticsearch.client.Requests#getTaskRequest() + */ + void getTask(GetTaskRequest request, ActionListener listener); + + /** + * Fetch a task by id. + */ + GetTaskRequestBuilder prepareGetTask(String taskId); + + /** + * Fetch a task by id. + */ + GetTaskRequestBuilder prepareGetTask(TaskId taskId); + /** * Cancel tasks * diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java index 276bd9d9062..6d652bf39d0 100644 --- a/core/src/main/java/org/elasticsearch/client/Requests.java +++ b/core/src/main/java/org/elasticsearch/client/Requests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; @@ -406,6 +407,16 @@ public class Requests { return new ListTasksRequest(); } + /** + * Creates a get task request. + * + * @return The nodes tasks request + * @see org.elasticsearch.client.ClusterAdminClient#getTask(GetTaskRequest) + */ + public static GetTaskRequest getTaskRequest() { + return new GetTaskRequest(); + } + /** * Creates a nodes tasks request against one or more nodes. Pass null or an empty array for all nodes. * diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index 6481bec1b83..8ca6d502e3c 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -49,6 +49,10 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksActio import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequestBuilder; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequestBuilder; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder; @@ -109,6 +113,18 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder; import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction; +import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequestBuilder; +import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction; +import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequestBuilder; +import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptResponse; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder; @@ -272,18 +288,6 @@ import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction; -import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequestBuilder; -import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequestBuilder; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptResponse; import org.elasticsearch.action.ingest.DeletePipelineAction; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.DeletePipelineRequestBuilder; @@ -339,6 +343,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import java.util.Map; @@ -851,6 +856,25 @@ public abstract class AbstractClient extends AbstractComponent implements Client return new ListTasksRequestBuilder(this, ListTasksAction.INSTANCE).setNodesIds(nodesIds); } + @Override + public ActionFuture getTask(final GetTaskRequest request) { + return execute(GetTaskAction.INSTANCE, request); + } + + @Override + public void getTask(final GetTaskRequest request, final ActionListener listener) { + execute(GetTaskAction.INSTANCE, request, listener); + } + + @Override + public GetTaskRequestBuilder prepareGetTask(String taskId) { + return prepareGetTask(new TaskId(taskId)); + } + + @Override + public GetTaskRequestBuilder prepareGetTask(TaskId taskId) { + return new GetTaskRequestBuilder(this, GetTaskAction.INSTANCE).setTaskId(taskId); + } @Override public ActionFuture cancelTasks(CancelTasksRequest request) { diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index d3a42a97ebb..efd525d313b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -148,18 +148,11 @@ public class ClusterChangedEvent { * has changed between the previous cluster state and the new cluster state. * Note that this is an object reference equality test, not an equals test. */ - public boolean indexMetaDataChanged(IndexMetaData current) { - MetaData previousMetaData = previousState.metaData(); - if (previousMetaData == null) { - return true; - } - IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex()); + public static boolean indexMetaDataChanged(IndexMetaData metaData1, IndexMetaData metaData2) { + assert metaData1 != null && metaData2 != null; // no need to check on version, since disco modules will make sure to use the // same instance if its a version match - if (previousIndexMetaData == current) { - return false; - } - return true; + return metaData1 != metaData2; } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index b9084d52a91..c90dbcd3c54 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -63,7 +63,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ExtensionPoint; import org.elasticsearch.gateway.GatewayAllocator; -import org.elasticsearch.tasks.TaskResultsService; +import org.elasticsearch.tasks.TaskPersistenceService; import java.util.Arrays; import java.util.Collections; @@ -156,6 +156,6 @@ public class ClusterModule extends AbstractModule { bind(ShardStateAction.class).asEagerSingleton(); bind(NodeMappingRefreshAction.class).asEagerSingleton(); bind(MappingUpdatedAction.class).asEagerSingleton(); - bind(TaskResultsService.class).asEagerSingleton(); + bind(TaskPersistenceService.class).asEagerSingleton(); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java index 0645accb42a..b1bf01018c9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaDataMappingService; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -58,13 +59,12 @@ public class NodeMappingRefreshAction extends AbstractComponent { transportService.registerRequestHandler(ACTION_NAME, NodeMappingRefreshRequest::new, ThreadPool.Names.SAME, new NodeMappingRefreshTransportHandler()); } - public void nodeMappingRefresh(final ClusterState state, final NodeMappingRefreshRequest request) { - final DiscoveryNodes nodes = state.nodes(); - if (nodes.getMasterNode() == null) { + public void nodeMappingRefresh(final DiscoveryNode masterNode, final NodeMappingRefreshRequest request) { + if (masterNode == null) { logger.warn("can't send mapping refresh for [{}], no master known.", request.index()); return; } - transportService.sendRequest(nodes.getMasterNode(), ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); + transportService.sendRequest(masterNode, ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME); } private class NodeMappingRefreshTransportHandler implements TransportRequestHandler { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index a1e891bce3d..ceea83dbcaa 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -375,21 +375,22 @@ public class IndexShardRoutingTable implements Iterable { return new PlainShardIterator(shardId, ordered); } - public ShardIterator preferNodeActiveInitializingShardsIt(String nodeId) { - ArrayList ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size()); + public ShardIterator preferNodeActiveInitializingShardsIt(Set nodeIds) { + ArrayList preferred = new ArrayList<>(activeShards.size() + allInitializingShards.size()); + ArrayList notPreferred = new ArrayList<>(activeShards.size() + allInitializingShards.size()); // fill it in a randomized fashion for (ShardRouting shardRouting : shuffler.shuffle(activeShards)) { - ordered.add(shardRouting); - if (nodeId.equals(shardRouting.currentNodeId())) { - // switch, its the matching node id - ordered.set(ordered.size() - 1, ordered.get(0)); - ordered.set(0, shardRouting); + if (nodeIds.contains(shardRouting.currentNodeId())) { + preferred.add(shardRouting); + } else { + notPreferred.add(shardRouting); } } + preferred.addAll(notPreferred); if (!allInitializingShards.isEmpty()) { - ordered.addAll(allInitializingShards); + preferred.addAll(allInitializingShards); } - return new PlainShardIterator(shardId, ordered); + return new PlainShardIterator(shardId, preferred); } @Override diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 9b1c82e7c06..129650de991 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -33,17 +33,15 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; -/** - * - */ public class OperationRouting extends AbstractComponent { - private final AwarenessAllocationDecider awarenessAllocationDecider; @Inject @@ -52,11 +50,11 @@ public class OperationRouting extends AbstractComponent { this.awarenessAllocationDecider = awarenessAllocationDecider; } - public ShardIterator indexShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing) { + public ShardIterator indexShards(ClusterState clusterState, String index, String id, @Nullable String routing) { return shards(clusterState, index, id, routing).shardsIt(); } - public ShardIterator getShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing, @Nullable String preference) { + public ShardIterator getShards(ClusterState clusterState, String index, String id, @Nullable String routing, @Nullable String preference) { return preferenceActiveShardIterator(shards(clusterState, index, id, routing), clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference); } @@ -158,10 +156,14 @@ public class OperationRouting extends AbstractComponent { } preferenceType = Preference.parse(preference); switch (preferenceType) { - case PREFER_NODE: - return indexShard.preferNodeActiveInitializingShardsIt(preference.substring(Preference.PREFER_NODE.type().length() + 1)); + case PREFER_NODES: + final Set nodesIds = + Arrays.stream( + preference.substring(Preference.PREFER_NODES.type().length() + 1).split(",") + ).collect(Collectors.toSet()); + return indexShard.preferNodeActiveInitializingShardsIt(nodesIds); case LOCAL: - return indexShard.preferNodeActiveInitializingShardsIt(localNodeId); + return indexShard.preferNodeActiveInitializingShardsIt(Collections.singleton(localNodeId)); case PRIMARY: return indexShard.primaryActiveInitializingShardIt(); case REPLICA: diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java index 6de251b9d52..cf0dd6cc54d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java @@ -30,9 +30,9 @@ public enum Preference { SHARDS("_shards"), /** - * Route to preferred node, if possible + * Route to preferred nodes, if possible */ - PREFER_NODE("_prefer_node"), + PREFER_NODES("_prefer_nodes"), /** * Route to local node, if possible @@ -98,8 +98,8 @@ public enum Preference { switch (preferenceType) { case "_shards": return SHARDS; - case "_prefer_node": - return PREFER_NODE; + case "_prefer_nodes": + return PREFER_NODES; case "_only_node": return ONLY_NODE; case "_local": @@ -123,6 +123,7 @@ public enum Preference { throw new IllegalArgumentException("no Preference for [" + preferenceType + "]"); } } + } diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java index 4087704d5cd..9982a08f17f 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java @@ -19,7 +19,7 @@ package org.elasticsearch.common.geo; import java.util.ArrayList; import java.util.Collection; -import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.BitUtil; /** @@ -39,7 +39,7 @@ public class GeoHashUtils { /** maximum precision for geohash strings */ public static final int PRECISION = 12; - private static final short MORTON_OFFSET = (GeoEncodingUtils.BITS<<1) - (PRECISION*5); + private static final short MORTON_OFFSET = (GeoPointField.BITS<<1) - (PRECISION*5); // No instance: private GeoHashUtils() { @@ -51,7 +51,7 @@ public class GeoHashUtils { public static final long longEncode(final double lon, final double lat, final int level) { // shift to appropriate level final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET); - return ((BitUtil.flipFlop(GeoEncodingUtils.mortonHash(lat, lon)) >>> msf) << 4) | level; + return ((BitUtil.flipFlop(GeoPointField.encodeLatLon(lat, lon)) >>> msf) << 4) | level; } /** @@ -117,7 +117,7 @@ public class GeoHashUtils { */ public static final String stringEncode(final double lon, final double lat, final int level) { // convert to geohashlong - final long ghLong = fromMorton(GeoEncodingUtils.mortonHash(lat, lon), level); + final long ghLong = fromMorton(GeoPointField.encodeLatLon(lat, lon), level); return stringEncode(ghLong); } @@ -138,7 +138,7 @@ public class GeoHashUtils { StringBuilder geoHash = new StringBuilder(); short precision = 0; - final short msf = (GeoEncodingUtils.BITS<<1)-5; + final short msf = (GeoPointField.BITS<<1)-5; long mask = 31L<>>(msf-(precision*5)))]); diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java index 5d1250a5148..96fe2826da8 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java @@ -19,12 +19,11 @@ package org.elasticsearch.common.geo; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.BitUtil; import static org.elasticsearch.common.geo.GeoHashUtils.mortonEncode; import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLat; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLon; /** * @@ -84,14 +83,14 @@ public final class GeoPoint { } public GeoPoint resetFromIndexHash(long hash) { - lon = mortonUnhashLon(hash); - lat = mortonUnhashLat(hash); + lon = GeoPointField.decodeLongitude(hash); + lat = GeoPointField.decodeLatitude(hash); return this; } public GeoPoint resetFromGeoHash(String geohash) { final long hash = mortonEncode(geohash); - return this.reset(mortonUnhashLat(hash), mortonUnhashLon(hash)); + return this.reset(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); } public GeoPoint resetFromGeoHash(long geohashLong) { @@ -164,8 +163,4 @@ public final class GeoPoint { public static GeoPoint fromGeohash(long geohashLong) { return new GeoPoint().resetFromGeoHash(geohashLong); } - - public static GeoPoint fromIndexLong(long indexLong) { - return new GeoPoint().resetFromIndexHash(indexLong); - } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index d5cc6846865..69ab2059ccf 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; -import static org.apache.lucene.spatial.util.GeoDistanceUtils.maxRadialDistanceMeters; import java.io.IOException; @@ -67,6 +66,9 @@ public class GeoUtils { /** Earth ellipsoid polar distance in meters */ public static final double EARTH_POLAR_DISTANCE = Math.PI * EARTH_SEMI_MINOR_AXIS; + /** rounding error for quantized latitude and longitude values */ + public static final double TOLERANCE = 1E-6; + /** Returns the minimum between the provided distance 'initialRadius' and the * maximum distance/radius from the point 'center' before overlapping **/ @@ -468,6 +470,14 @@ public class GeoUtils { } } + /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */ + public static double maxRadialDistanceMeters(final double centerLat, final double centerLon) { + if (Math.abs(centerLat) == MAX_LAT) { + return SloppyMath.haversinMeters(centerLat, centerLon, 0, centerLon); + } + return SloppyMath.haversinMeters(centerLat, centerLon, centerLat, (MAX_LON + centerLon) % 360); + } + private GeoUtils() { } } diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java index 4added7df31..a37c6371482 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java @@ -26,6 +26,8 @@ import java.io.IOException; * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization. + * {@linkplain org.elasticsearch.common.unit.TimeValue} actually implements {@linkplain Writeable} not {@linkplain Streamable} but it has + * the same contract. * * Prefer implementing {@link Writeable} over implementing this interface where possible. Lots of code depends on this interface so this * isn't always possible. diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java index 55ab419881f..cf127e5b968 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java @@ -26,8 +26,6 @@ import java.io.IOException; * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization. - * {@linkplain org.elasticsearch.common.unit.TimeValue} actually implements {@linkplain Streamable} not {@linkplain Writeable} but it has - * the same contract. * * Prefer implementing this interface over implementing {@link Streamable} where possible. Lots of code depends on {@linkplain Streamable} * so this isn't always possible. diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index 9b995f423a3..75f400fdc9d 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -45,6 +45,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.SmallFloat; import java.io.IOException; +import java.util.Objects; import java.util.Set; /** @@ -63,6 +64,19 @@ public final class AllTermQuery extends Query { this.term = term; } + @Override + public boolean equals(Object obj) { + if (sameClassAs(obj) == false) { + return false; + } + return Objects.equals(term, ((AllTermQuery) obj).term); + } + + @Override + public int hashCode() { + return 31 * classHash() + term.hashCode(); + } + @Override public Query rewrite(IndexReader reader) throws IOException { Query rewritten = super.rewrite(reader); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java index a25b4c0aa29..9caf350926c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java @@ -66,4 +66,14 @@ public class MatchNoDocsQuery extends Query { public String toString(String field) { return "MatchNoDocsQuery[\"" + reason + "\"]"; } + + @Override + public boolean equals(Object obj) { + return sameClassAs(obj); + } + + @Override + public int hashCode() { + return classHash(); + } } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java index fbe0c28e341..06ab2b4a530 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java @@ -84,14 +84,14 @@ public class MoreLikeThisQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText), + return Objects.hash(classHash(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText), maxDocFreq, maxQueryTerms, maxWordLen, minDocFreq, minTermFrequency, minWordLen, Arrays.hashCode(moreLikeFields), minimumShouldMatch, stopWords); } @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } MoreLikeThisQuery other = (MoreLikeThisQuery) obj; diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java index 05006ec0db7..87bfdacb1c7 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -238,7 +238,7 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public boolean equals(Object o) { - if (super.equals(o) == false) { + if (sameClassAs(o) == false) { return false; } MultiPhrasePrefixQuery other = (MultiPhrasePrefixQuery) o; @@ -252,7 +252,7 @@ public class MultiPhrasePrefixQuery extends Query { */ @Override public int hashCode() { - return super.hashCode() + return classHash() ^ slop ^ termArraysHashCode() ^ positions.hashCode(); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index e62f3f6665a..3927dcd518e 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -355,7 +355,7 @@ public class FiltersFunctionScoreQuery extends Query { if (this == o) { return true; } - if (super.equals(o) == false) { + if (sameClassAs(o) == false) { return false; } FiltersFunctionScoreQuery other = (FiltersFunctionScoreQuery) o; @@ -367,6 +367,6 @@ public class FiltersFunctionScoreQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), subQuery, maxBoost, combineFunction, minScore, scoreMode, Arrays.hashCode(filterFunctions)); + return Objects.hash(classHash(), subQuery, maxBoost, combineFunction, minScore, scoreMode, Arrays.hashCode(filterFunctions)); } } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 646076a3a17..be98a07a9c1 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -210,7 +210,7 @@ public class FunctionScoreQuery extends Query { if (this == o) { return true; } - if (super.equals(o) == false) { + if (sameClassAs(o) == false) { return false; } FunctionScoreQuery other = (FunctionScoreQuery) o; @@ -221,6 +221,6 @@ public class FunctionScoreQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), subQuery.hashCode(), function, combineFunction, minScore, maxBoost); + return Objects.hash(classHash(), subQuery.hashCode(), function, combineFunction, minScore, maxBoost); } } diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java index fd9191fd713..1484ad64018 100644 --- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java +++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java @@ -49,6 +49,7 @@ import org.elasticsearch.rest.action.admin.cluster.node.hotthreads.RestNodesHotT import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction; import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction; import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestCancelTasksAction; +import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestGetTaskAction; import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction; import org.elasticsearch.rest.action.admin.cluster.repositories.delete.RestDeleteRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction; @@ -146,6 +147,7 @@ import org.elasticsearch.rest.action.suggest.RestSuggestAction; import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction; import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction; import org.elasticsearch.rest.action.update.RestUpdateAction; +import org.elasticsearch.tasks.RawTaskStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; @@ -277,6 +279,7 @@ public class NetworkModule extends AbstractModule { // Tasks API RestListTasksAction.class, + RestGetTaskAction.class, RestCancelTasksAction.class, // Ingest API @@ -339,6 +342,7 @@ public class NetworkModule extends AbstractModule { registerTransport(LOCAL_TRANSPORT, LocalTransport.class); registerTransport(NETTY_TRANSPORT, NettyTransport.class); registerTaskStatus(ReplicationTask.Status.NAME, ReplicationTask.Status::new); + registerTaskStatus(RawTaskStatus.NAME, RawTaskStatus::new); registerBuiltinAllocationCommands(); if (transportClient == false) { diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java index c78d6c730b6..593b0484800 100644 --- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java +++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; import org.joda.time.DateTimeField; import org.joda.time.DateTimeZone; +import org.joda.time.IllegalInstantException; import java.io.IOException; import java.util.Objects; @@ -218,7 +219,56 @@ public abstract class TimeZoneRounding extends Rounding { public long roundKey(long utcMillis) { long timeLocal = timeZone.convertUTCToLocal(utcMillis); long rounded = Rounding.Interval.roundValue(Rounding.Interval.roundKey(timeLocal, interval), interval); - return timeZone.convertLocalToUTC(rounded, false, utcMillis); + long roundedUTC; + if (isInDSTGap(rounded) == false) { + roundedUTC = timeZone.convertLocalToUTC(rounded, true, utcMillis); + } else { + /* + * Edge case where the rounded local time is illegal and landed + * in a DST gap. In this case, we choose 1ms tick after the + * transition date. We don't want the transition date itself + * because those dates, when rounded themselves, fall into the + * previous interval. This would violate the invariant that the + * rounding operation should be idempotent. + */ + roundedUTC = timeZone.previousTransition(utcMillis) + 1; + } + return roundedUTC; + } + + /** + * Determine whether the local instant is a valid instant in the given + * time zone. The logic for this is taken from + * {@link DateTimeZone#convertLocalToUTC(long, boolean)} for the + * `strict` mode case, but instead of throwing an + * {@link IllegalInstantException}, which is costly, we want to return a + * flag indicating that the value is illegal in that time zone. + */ + private boolean isInDSTGap(long instantLocal) { + if (timeZone.isFixed()) { + return false; + } + // get the offset at instantLocal (first estimate) + int offsetLocal = timeZone.getOffset(instantLocal); + // adjust instantLocal using the estimate and recalc the offset + int offset = timeZone.getOffset(instantLocal - offsetLocal); + // if the offsets differ, we must be near a DST boundary + if (offsetLocal != offset) { + // determine if we are in the DST gap + long nextLocal = timeZone.nextTransition(instantLocal - offsetLocal); + if (nextLocal == (instantLocal - offsetLocal)) { + nextLocal = Long.MAX_VALUE; + } + long nextAdjusted = timeZone.nextTransition(instantLocal - offset); + if (nextAdjusted == (instantLocal - offset)) { + nextAdjusted = Long.MAX_VALUE; + } + if (nextLocal != nextAdjusted) { + // we are in the DST gap + return true; + } + } + return false; } @Override diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 6eb8df68242..8c2d4dc01bf 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -181,6 +181,7 @@ public final class ClusterSettings extends AbstractScopedSettings { IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING, + IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING, IndicesTTLService.INDICES_TTL_INTERVAL_SETTING, MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING, MetaData.SETTING_READ_ONLY_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index b9d0c6b4c70..fb60453d467 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -116,6 +116,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.ALLOW_UNMAPPED, IndexSettings.INDEX_CHECK_ON_STARTUP, IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, + IndexSettings.MAX_SLICES_PER_SCROLL, ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING, IndexSettings.INDEX_GC_DELETES_SETTING, IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java index c467a0c18a8..2058355d300 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java @@ -23,7 +23,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.io.stream.Writeable; import org.joda.time.Period; import org.joda.time.PeriodType; import org.joda.time.format.PeriodFormat; @@ -34,7 +34,7 @@ import java.util.Locale; import java.util.Objects; import java.util.concurrent.TimeUnit; -public class TimeValue implements Streamable { +public class TimeValue implements Writeable { /** How many nano-seconds in one milli-second */ public static final long NSEC_PER_MSEC = 1000000; @@ -59,13 +59,8 @@ public class TimeValue implements Streamable { return new TimeValue(hours, TimeUnit.HOURS); } - private long duration; - - private TimeUnit timeUnit; - - private TimeValue() { - - } + private final long duration; + private final TimeUnit timeUnit; public TimeValue(long millis) { this(millis, TimeUnit.MILLISECONDS); @@ -76,6 +71,19 @@ public class TimeValue implements Streamable { this.timeUnit = timeUnit; } + /** + * Read from a stream. + */ + public TimeValue(StreamInput in) throws IOException { + duration = in.readZLong(); + timeUnit = TimeUnit.NANOSECONDS; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeZLong(nanos()); + } + public long nanos() { return timeUnit.toNanos(duration); } @@ -304,26 +312,6 @@ public class TimeValue implements Streamable { static final long C5 = C4 * 60L; static final long C6 = C5 * 24L; - public static TimeValue readTimeValue(StreamInput in) throws IOException { - TimeValue timeValue = new TimeValue(); - timeValue.readFrom(in); - return timeValue; - } - - /** - * serialization converts TimeValue internally to NANOSECONDS - */ - @Override - public void readFrom(StreamInput in) throws IOException { - duration = in.readLong(); - timeUnit = TimeUnit.NANOSECONDS; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeLong(nanos()); - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java index 0fb20aa519b..6f8a606d9a6 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java @@ -21,7 +21,9 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import java.io.IOException; import java.util.ArrayList; @@ -123,6 +125,17 @@ public abstract class AbstractObjectParser parseArray(p, p::intValue), field, ValueType.INT_ARRAY); } + public void declareRawObject(BiConsumer consumer, ParseField field) { + NoContextParser bytesParser = p -> { + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.prettyPrint(); + builder.copyCurrentStructure(p); + return builder.bytes(); + } + }; + declareField(consumer, bytesParser, field, ValueType.OBJECT); + } + private interface IOSupplier { T get() throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 56baf30c9e0..f86b44b4e6f 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -328,6 +328,10 @@ public class JsonXContentGenerator implements XContentGenerator { if (mayWriteRawData(contentType) == false) { copyRawValue(content, contentType.xContent()); } else { + if (generator.getOutputContext().getCurrentName() != null) { + // If we've just started a field we'll need to add the separator + generator.writeRaw(':'); + } flush(); content.writeTo(os); writeEndRaw(); diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java index 5f101cf3a1f..d6d523508e1 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java @@ -45,6 +45,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -196,14 +197,15 @@ public class MasterFaultDetection extends FaultDetection { private void notifyMasterFailure(final DiscoveryNode masterNode, final Throwable cause, final String reason) { if (notifiedMasterFailure.compareAndSet(false, true)) { - threadPool.generic().execute(new Runnable() { - @Override - public void run() { + try { + threadPool.generic().execute(() -> { for (Listener listener : listeners) { listener.onMasterFailure(masterNode, cause, reason); } - } - }); + }); + } catch (RejectedExecutionException e) { + logger.error("master failure notification was rejected, it's highly likely the node is shutting down", e); + } stop("master failure, " + reason); } } diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java index d200ca5b07b..1b4f1ba75b0 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java @@ -20,6 +20,7 @@ package org.elasticsearch.discovery.zen.ping.unicast; import com.carrotsearch.hppc.cursors.ObjectCursor; + import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; @@ -79,7 +80,6 @@ import java.util.function.Function; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap; import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPingResponse; @@ -545,7 +545,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implemen public void readFrom(StreamInput in) throws IOException { super.readFrom(in); id = in.readInt(); - timeout = readTimeValue(in); + timeout = new TimeValue(in); pingResponse = readPingResponse(in); } diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java index ccc90d9149c..31bbd235009 100644 --- a/core/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java @@ -108,7 +108,11 @@ public class HttpServer extends AbstractLifecycleComponent implement RestChannel responseChannel = channel; try { int contentLength = request.content().length(); - inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); + if (restController.canTripCircuitBreaker(request)) { + inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, ""); + } else { + inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); + } // iff we could reserve bytes for the request we need to send the response also over this channel responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength); restController.dispatchRequest(request, responseChannel, threadContext); diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 5780dc256a5..1902f24c4b7 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; +import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.threadpool.ThreadPool; @@ -93,7 +94,7 @@ import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; /** * */ -public final class IndexService extends AbstractIndexComponent implements IndexComponent, Iterable { +public class IndexService extends AbstractIndexComponent implements IndicesClusterStateService.AllocatedIndex { private final IndexEventListener eventListener; private final AnalysisService analysisService; @@ -184,8 +185,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC /** * Return the shard with the provided id, or null if there is no such shard. */ - @Nullable - public IndexShard getShardOrNull(int shardId) { + @Override + public @Nullable IndexShard getShardOrNull(int shardId) { return shards.get(shardId); } @@ -359,6 +360,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC return primary == false && IndexMetaData.isIndexUsingShadowReplicas(indexSettings); } + @Override public synchronized void removeShard(int shardId, String reason) { final ShardId sId = new ShardId(index(), shardId); final IndexShard indexShard; @@ -470,6 +472,11 @@ public final class IndexService extends AbstractIndexComponent implements IndexC return searchOperationListeners; } + @Override + public boolean updateMapping(IndexMetaData indexMetaData) throws IOException { + return mapperService().updateMapping(indexMetaData); + } + private class StoreCloseListener implements Store.OnClose { private final ShardId shardId; private final boolean ownsShard; @@ -617,6 +624,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC return indexSettings.getIndexMetaData(); } + @Override public synchronized void updateMetaData(final IndexMetaData metadata) { final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability(); if (indexSettings.updateIndexMetaData(metadata)) { diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index 592c1ff1125..2c20697d757 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -121,6 +121,12 @@ public final class IndexSettings { public static final Setting MAX_REFRESH_LISTENERS_PER_SHARD = Setting.intSetting("index.max_refresh_listeners", 1000, 0, Property.Dynamic, Property.IndexScope); + /** + * The maximum number of slices allowed in a scroll request + */ + public static final Setting MAX_SLICES_PER_SCROLL = Setting.intSetting("index.max_slices_per_scroll", + 1024, 1, Property.Dynamic, Property.IndexScope); + private final Index index; private final Version version; private final ESLogger logger; @@ -154,6 +160,11 @@ public final class IndexSettings { * The maximum number of refresh listeners allows on this shard. */ private volatile int maxRefreshListeners; + /** + * The maximum number of slices allowed in a scroll request. + */ + private volatile int maxSlicesPerScroll; + /** * Returns the default search field for this index. @@ -239,6 +250,7 @@ public final class IndexSettings { maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING); TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING); maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD); + maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL); this.mergePolicyConfig = new MergePolicyConfig(logger, this); assert indexNameMatcher.test(indexMetaData.getIndex().getName()); @@ -262,6 +274,7 @@ public final class IndexSettings { scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize); scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval); scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners); + scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll); } private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) { @@ -391,7 +404,7 @@ public final class IndexSettings { * * @return true iff any setting has been updated otherwise false. */ - synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) { + public synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) { final Settings newSettings = indexMetaData.getSettings(); if (version.equals(Version.indexCreated(newSettings)) == false) { throw new IllegalArgumentException("version mismatch on settings update expected: " + version + " but was: " + Version.indexCreated(newSettings)); @@ -521,5 +534,16 @@ public final class IndexSettings { this.maxRefreshListeners = maxRefreshListeners; } + /** + * The maximum number of slices allowed in a scroll request. + */ + public int getMaxSlicesPerScroll() { + return maxSlicesPerScroll; + } + + private void setMaxSlicesPerScroll(int value) { + this.maxSlicesPerScroll = value; + } + IndexScopedSettings getScopedSettings() { return scopedSettings;} } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java index 949e1ef0fb4..db428db153d 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; @@ -35,10 +36,11 @@ public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory { public PatternReplaceCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name); - if (!Strings.hasLength(settings.get("pattern"))) { + String sPattern = settings.get("pattern"); + if (!Strings.hasLength(sPattern)) { throw new IllegalArgumentException("pattern is missing for [" + name + "] char filter of type 'pattern_replace'"); } - pattern = Pattern.compile(settings.get("pattern")); + pattern = Regex.compile(sPattern, settings.get("flags")); replacement = settings.get("replacement", ""); // when not set or set to "", use "". } diff --git a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java index ef81b288f92..c955cbef75d 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java @@ -20,15 +20,10 @@ package org.elasticsearch.index.cache.request; import org.apache.lucene.util.Accountable; -import org.elasticsearch.common.cache.RemovalListener; -import org.elasticsearch.common.cache.RemovalNotification; import org.elasticsearch.common.metrics.CounterMetric; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.IndicesRequestCache; /** + * Tracks the portion of the request cache in use for a particular shard. */ public final class ShardRequestCache { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index e3dc84a3477..967d07174b9 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; @@ -426,7 +425,7 @@ public final class OrdinalsBuilder implements Closeable { protected AcceptStatus accept(BytesRef term) throws IOException { // accept only the max resolution terms // todo is this necessary? - return GeoEncodingUtils.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ? + return GeoPointField.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ? AcceptStatus.YES : AcceptStatus.END; } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java index c18f96c06b0..90554bd1308 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.CharsRefBuilder; @@ -58,7 +57,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData cursor : indexMetaData.getMappings().values()) { + MappingMetaData mappingMd = cursor.value; + String mappingType = mappingMd.type(); + CompressedXContent mappingSource = mappingMd.source(); + // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same + // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the + // merge version of it, which it does when refreshing the mappings), and warn log it. + try { + DocumentMapper existingMapper = documentMapper(mappingType); + + if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) { + String op = existingMapper == null ? "adding" : "updating"; + if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) { + logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string()); + } else if (logger.isTraceEnabled()) { + logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string()); + } else { + logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op, + mappingType); + } + merge(mappingType, mappingSource, MergeReason.MAPPING_RECOVERY, true); + if (!documentMapper(mappingType).mappingSource().equals(mappingSource)) { + logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(), + mappingType, mappingSource, documentMapper(mappingType).mappingSource()); + requireRefresh = true; + } + } + } catch (Throwable e) { + logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index(), mappingType, mappingSource); + throw e; + } + } + return requireRefresh; + } + //TODO: make this atomic public void merge(Map> mappings, boolean updateAllTypes) throws MapperParsingException { // first, add the default mapping diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java index 831f4575f3d..2a8938b4ab7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java @@ -35,7 +35,6 @@ public final class Uid { public static final char DELIMITER = '#'; public static final byte DELIMITER_BYTE = 0x23; - public static final BytesRef DELIMITER_BYTES = new BytesRef(new byte[]{DELIMITER_BYTE}); private final String type; @@ -83,13 +82,6 @@ public final class Uid { return createUidAsBytes(type, id); } - public static BytesRef typePrefixAsBytes(BytesRef type) { - BytesRefBuilder bytesRef = new BytesRefBuilder(); - bytesRef.append(type); - bytesRef.append(DELIMITER_BYTES); - return bytesRef.toBytesRef(); - } - public static Uid createUid(String uid) { int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1)); @@ -136,35 +128,4 @@ public final class Uid { return type + DELIMITER + id; } - public static boolean hasDelimiter(BytesRef uid) { - final int limit = uid.offset + uid.length; - for (int i = uid.offset; i < limit; i++) { - if (uid.bytes[i] == DELIMITER_BYTE) { // 0x23 is equal to '#' - return true; - } - } - return false; - } - - public static BytesRef[] splitUidIntoTypeAndId(BytesRef uid) { - int loc = -1; - final int limit = uid.offset + uid.length; - for (int i = uid.offset; i < limit; i++) { - if (uid.bytes[i] == DELIMITER_BYTE) { // 0x23 is equal to '#' - loc = i; - break; - } - } - - if (loc == -1) { - return null; - } - - int idStart = loc + 1; - return new BytesRef[] { - new BytesRef(uid.bytes, uid.offset, loc - uid.offset), - new BytesRef(uid.bytes, idStart, limit - idStart) - }; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index a79631481d2..66cb7255fd6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -204,7 +204,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu @Override public boolean equals(Object o) { if (this == o) return true; - if (!super.equals(o)) return false; + if (sameClassAs(o) == false) return false; LateParsingQuery that = (LateParsingQuery) o; if (includeLower != that.includeLower) return false; @@ -218,7 +218,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu @Override public int hashCode() { - return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); + return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java index a7e44ba3654..a3374153955 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java @@ -213,7 +213,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper { @Override public boolean equals(Object o) { if (this == o) return true; - if (!super.equals(o)) return false; + if (sameClassAs(o) == false) return false; LateParsingQuery that = (LateParsingQuery) o; if (includeLower != that.includeLower) return false; @@ -227,7 +227,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper { @Override public int hashCode() { - return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); + return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index c72d746158b..8e6b538a2ae 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -239,6 +239,13 @@ public class TypeParsers { Map.Entry entry = iterator.next(); final String propName = entry.getKey(); final Object propNode = entry.getValue(); + if (false == propName.equals("null_value") && propNode == null) { + /* + * No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather + * than send them a null pointer exception later. + */ + throw new MapperParsingException("[" + propName + "] must not have a [null] value"); + } if (propName.equals("store")) { builder.store(parseStore(name, propNode.toString(), parserContext)); iterator.remove(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index f960ecaa977..d882be8e9d7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -195,7 +195,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } TypeQuery that = (TypeQuery) obj; @@ -204,7 +204,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { @Override public int hashCode() { - return 31 * super.hashCode() + type.hashCode(); + return 31 * classHash() + type.hashCode(); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 1e2a078925f..a123f64c4d6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -23,7 +23,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; -import org.apache.lucene.document.XInetAddressPoint; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.XPointValues; @@ -176,7 +175,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include if (fields.length == 2) { InetAddress address = InetAddresses.forString(fields[0]); int prefixLength = Integer.parseInt(fields[1]); - return XInetAddressPoint.newPrefixQuery(name(), address, prefixLength); + return InetAddressPoint.newPrefixQuery(name(), address, prefixLength); } else { throw new IllegalArgumentException("Expected [ip/prefix] but was [" + term + "]"); } @@ -191,27 +190,27 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include failIfNotIndexed(); InetAddress lower; if (lowerTerm == null) { - lower = XInetAddressPoint.MIN_VALUE; + lower = InetAddressPoint.MIN_VALUE; } else { lower = parse(lowerTerm); if (includeLower == false) { - if (lower.equals(XInetAddressPoint.MAX_VALUE)) { + if (lower.equals(InetAddressPoint.MAX_VALUE)) { return new MatchNoDocsQuery(); } - lower = XInetAddressPoint.nextUp(lower); + lower = InetAddressPoint.nextUp(lower); } } InetAddress upper; if (upperTerm == null) { - upper = XInetAddressPoint.MAX_VALUE; + upper = InetAddressPoint.MAX_VALUE; } else { upper = parse(upperTerm); if (includeUpper == false) { - if (upper.equals(XInetAddressPoint.MIN_VALUE)) { + if (upper.equals(InetAddressPoint.MIN_VALUE)) { return new MatchNoDocsQuery(); } - upper = XInetAddressPoint.nextDown(upper); + upper = InetAddressPoint.nextDown(upper); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java index 06f30a3477e..cf18d5a4c10 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java @@ -23,7 +23,6 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.spatial.geopoint.search.XGeoPointDistanceRangeQuery; -import org.apache.lucene.spatial.util.GeoDistanceUtils; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -48,8 +47,6 @@ import java.util.Locale; import java.util.Objects; import java.util.Optional; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE; - public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "geo_distance_range"; @@ -354,7 +351,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder public boolean equals(Object obj) { if (this == obj) return true; - if (!super.equals(obj)) + if (sameClassAs(obj) == false) return false; ScriptQuery other = (ScriptQuery) obj; return Objects.equals(script, other.script); @@ -192,7 +192,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder @Override public int hashCode() { - return Objects.hash(super.hashCode(), script); + return Objects.hash(classHash(), script); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java index 6f92e411c00..6c4fd23e64c 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java @@ -190,7 +190,7 @@ public class GeoDistanceRangeQuery extends Query { @Override public boolean equals(Object o) { if (this == o) return true; - if (super.equals(o) == false) return false; + if (sameClassAs(o) == false) return false; GeoDistanceRangeQuery filter = (GeoDistanceRangeQuery) o; @@ -212,7 +212,7 @@ public class GeoDistanceRangeQuery extends Query { @Override public int hashCode() { - int result = super.hashCode(); + int result = classHash(); long temp; temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L; result = 31 * result + Long.hashCode(temp); diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java index d62aa76efd9..c3a52cb114e 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java @@ -111,7 +111,7 @@ public class GeoPolygonQuery extends Query { @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } GeoPolygonQuery that = (GeoPolygonQuery) obj; @@ -121,7 +121,7 @@ public class GeoPolygonQuery extends Query { @Override public int hashCode() { - int h = super.hashCode(); + int h = classHash(); h = 31 * h + indexFieldData.getFieldName().hashCode(); h = 31 * h + Arrays.hashCode(points); return h; diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java index 2f2801a2abe..789ee25e1b5 100644 --- a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java @@ -84,7 +84,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query { @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } InMemoryGeoBoundingBoxQuery other = (InMemoryGeoBoundingBoxQuery) obj; @@ -95,7 +95,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), fieldName(), topLeft, bottomRight); + return Objects.hash(classHash(), fieldName(), topLeft, bottomRight); } private static class Meridian180GeoBoundingBoxBits implements Bits { diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java index 916cf563fb8..9082fc072da 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java +++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java @@ -62,13 +62,6 @@ public class CommitPoint { public String checksum() { return checksum; } - - public boolean isSame(StoreFileMetaData md) { - if (checksum == null || md.checksum() == null) { - return false; - } - return length == md.length() && checksum.equals(md.checksum()); - } } public static enum Type { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 105d1e5b057..26701703ce1 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -37,8 +37,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Booleans; @@ -108,6 +106,7 @@ import org.elasticsearch.index.warmer.ShardIndexWarmerService; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.IndexingMemoryController; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.indices.recovery.RecoveryTargetService; @@ -136,7 +135,7 @@ import java.util.function.Consumer; import java.util.function.BiConsumer; import java.util.stream.Collectors; -public class IndexShard extends AbstractIndexShardComponent { +public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard { private final ThreadPool threadPool; private final MapperService mapperService; @@ -146,7 +145,7 @@ public class IndexShard extends AbstractIndexShardComponent { private final ShardSearchStats searchStats = new ShardSearchStats(); private final ShardGetService getService; private final ShardIndexWarmerService shardWarmerService; - private final ShardRequestCache shardQueryCache; + private final ShardRequestCache requestCacheStats; private final ShardFieldData shardFieldData; private final IndexFieldDataService indexFieldDataService; private final ShardBitsetFilterCache shardBitsetFilterCache; @@ -242,7 +241,7 @@ public class IndexShard extends AbstractIndexShardComponent { this.searchOperationListener = new SearchOperationListener.CompositeListener(searchListenersList, logger); this.getService = new ShardGetService(indexSettings, this, mapperService); this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings); - this.shardQueryCache = new ShardRequestCache(); + this.requestCacheStats = new ShardRequestCache(); this.shardFieldData = new ShardFieldData(); this.indexFieldDataService = indexFieldDataService; this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings); @@ -304,7 +303,7 @@ public class IndexShard extends AbstractIndexShardComponent { } public ShardRequestCache requestCache() { - return this.shardQueryCache; + return this.requestCacheStats; } public ShardFieldData fieldData() { @@ -338,6 +337,7 @@ public class IndexShard extends AbstractIndexShardComponent { /** * Returns the latest cluster routing entry received with this shard. */ + @Override public ShardRouting routingEntry() { return this.shardRouting; } @@ -348,13 +348,12 @@ public class IndexShard extends AbstractIndexShardComponent { /** * Updates the shards routing entry. This mutate the shards internal state depending - * on the changes that get introduced by the new routing value. This method will persist shard level metadata - * unless explicitly disabled. + * on the changes that get introduced by the new routing value. This method will persist shard level metadata. * * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted * @throws IOException if shard state could not be persisted */ - public void updateRoutingEntry(final ShardRouting newRouting, final boolean persistState) throws IOException { + public void updateRoutingEntry(final ShardRouting newRouting) throws IOException { final ShardRouting currentRouting = this.shardRouting; if (!newRouting.shardId().equals(shardId())) { throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId() + ""); @@ -408,9 +407,7 @@ public class IndexShard extends AbstractIndexShardComponent { } this.shardRouting = newRouting; indexEventListener.shardRoutingChanged(this, currentRouting, newRouting); - if (persistState) { - persistMetadata(newRouting, currentRouting); - } + persistMetadata(newRouting, currentRouting); } /** @@ -589,7 +586,7 @@ public class IndexShard extends AbstractIndexShardComponent { */ public void refresh(String source) { verifyNotClosed(); - + if (canIndex()) { long bytes = getEngine().getIndexBufferRAMBytesUsed(); writingBytes.addAndGet(bytes); @@ -1370,35 +1367,36 @@ public class IndexShard extends AbstractIndexShardComponent { return this.currentEngineReference.get(); } - public void startRecovery(DiscoveryNode localNode, DiscoveryNode sourceNode, RecoveryTargetService recoveryTargetService, + public void startRecovery(RecoveryState recoveryState, RecoveryTargetService recoveryTargetService, RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, - BiConsumer mappingUpdateConsumer, IndicesService indicesService) { - final RestoreSource restoreSource = shardRouting.restoreSource(); - - if (shardRouting.isPeerRecovery()) { - assert sourceNode != null : "peer recovery started but sourceNode is null"; - // we don't mark this one as relocated at the end. - // For primaries: requests in any case are routed to both when its relocating and that way we handle - // the edge case where its mark as relocated, and we might need to roll it back... - // For replicas: we are recovering a backup from a primary - RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA; - RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(), type, sourceNode, localNode); - try { - markAsRecovering("from " + sourceNode, recoveryState); - recoveryTargetService.startRecovery(this, type, sourceNode, recoveryListener); - } catch (Throwable e) { - failShard("corrupted preexisting index", e); - recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, e), true); - } - } else if (restoreSource == null) { - // recover from filesystem store - - IndexMetaData indexMetaData = indexSettings().getIndexMetaData(); - Index mergeSourceIndex = indexMetaData.getMergeSourceIndex(); - final boolean recoverFromLocalShards = mergeSourceIndex != null && shardRouting.allocatedPostIndexCreate(indexMetaData) == false && shardRouting.primary(); - final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(), - recoverFromLocalShards ? RecoveryState.Type.LOCAL_SHARDS : RecoveryState.Type.STORE, localNode, localNode); - if (recoverFromLocalShards) { + BiConsumer mappingUpdateConsumer, + IndicesService indicesService) { + switch (recoveryState.getType()) { + case PRIMARY_RELOCATION: + case REPLICA: + try { + markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState); + recoveryTargetService.startRecovery(this, recoveryState.getType(), recoveryState.getSourceNode(), recoveryListener); + } catch (Throwable e) { + failShard("corrupted preexisting index", e); + recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true); + } + break; + case STORE: + markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread + threadPool.generic().execute(() -> { + try { + if (recoverFromStore()) { + recoveryListener.onRecoveryDone(recoveryState); + } + } catch (Throwable t) { + recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true); + } + }); + break; + case LOCAL_SHARDS: + final IndexMetaData indexMetaData = indexSettings().getIndexMetaData(); + final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex(); final List startedShards = new ArrayList<>(); final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex); final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1; @@ -1414,14 +1412,14 @@ public class IndexShard extends AbstractIndexShardComponent { threadPool.generic().execute(() -> { try { final Set shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(), - indexMetaData.getNumberOfShards()); + + indexMetaData.getNumberOfShards()); if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream() .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) { recoveryListener.onRecoveryDone(recoveryState); } } catch (Throwable t) { recoveryListener.onRecoveryFailure(recoveryState, - new RecoveryFailedException(shardId, localNode, localNode, t), true); + new RecoveryFailedException(recoveryState, null, t), true); } }); } else { @@ -1433,36 +1431,25 @@ public class IndexShard extends AbstractIndexShardComponent { + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId()); } - recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, localNode, localNode, t), true); + recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true); } - } else { - markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread + break; + case SNAPSHOT: + markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(() -> { try { - if (recoverFromStore()) { + final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository( + recoveryState.getRestoreSource().snapshot().getRepository()); + if (restoreFromRepository(indexShardRepository)) { recoveryListener.onRecoveryDone(recoveryState); } - } catch (Throwable t) { - recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, t), true); + } catch (Throwable first) { + recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, first), true); } - }); - } - } else { - // recover from a restore - final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(), - RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode); - markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread - threadPool.generic().execute(() -> { - try { - final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshot().getRepository()); - if (restoreFromRepository(indexShardRepository)) { - recoveryListener.onRecoveryDone(recoveryState); - } - } catch (Throwable first) { - recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, first), true); - } - }); + break; + default: + throw new IllegalArgumentException("Unknown recovery type " + recoveryState.getType()); } } @@ -1472,7 +1459,7 @@ public class IndexShard extends AbstractIndexShardComponent { // called by the current engine @Override public void onFailedEngine(String reason, @Nullable Throwable failure) { - final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure, getIndexUUID()); + final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure); for (Callback listener : delegates) { try { listener.handle(shardFailure); @@ -1661,13 +1648,11 @@ public class IndexShard extends AbstractIndexShardComponent { public final String reason; @Nullable public final Throwable cause; - public final String indexUUID; - public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause, String indexUUID) { + public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause) { this.routing = routing; this.reason = reason; this.cause = cause; - this.indexUUID = indexUUID; } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java index ab3e334714a..b594b31abb8 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java +++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java @@ -68,15 +68,16 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener * @param location the location to listen for * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with * false otherwise. + * @return did we call the listener (true) or register the listener to call later (false)? */ - public void addOrNotify(Translog.Location location, Consumer listener) { + public boolean addOrNotify(Translog.Location location, Consumer listener) { requireNonNull(listener, "listener cannot be null"); requireNonNull(location, "location cannot be null"); if (lastRefreshedLocation != null && lastRefreshedLocation.compareTo(location) >= 0) { // Location already visible, just call the listener listener.accept(false); - return; + return true; } synchronized (this) { if (refreshListeners == null) { @@ -85,12 +86,13 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) { // We have a free slot so register the listener refreshListeners.add(new Tuple<>(location, listener)); - return; + return false; } } // No free slot so force a refresh and call the listener in this thread forceRefresh.run(); listener.accept(true); + return true; } /** @@ -135,14 +137,14 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener */ return; } - // First check if we've actually moved forward. If not then just bail immediately. - assert lastRefreshedLocation == null || currentRefreshLocation.compareTo(lastRefreshedLocation) >= 0; - if (lastRefreshedLocation != null && currentRefreshLocation.compareTo(lastRefreshedLocation) == 0) { - return; - } /* * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing - * around with refreshListeners or synchronizing at all. + * around with refreshListeners or synchronizing at all. Note that it is not safe for us to abort early if we haven't advanced the + * position here because we set and read lastRefreshedLocation outside of a synchronized block. We do that so that waiting for a + * refresh that has already passed is just a volatile read but the cost is that any check whether or not we've advanced the + * position will introduce a race between adding the listener and the position check. We could work around this by moving this + * assignment into the synchronized block below and double checking lastRefreshedLocation in addOrNotify's synchronized block but + * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate. */ lastRefreshedLocation = currentRefreshLocation; /* diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java index e22f684637e..e35c95ae1f0 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java @@ -59,16 +59,16 @@ public final class ShadowIndexShard extends IndexShard { /** * In addition to the regular accounting done in - * {@link IndexShard#updateRoutingEntry(ShardRouting, boolean)}, + * {@link IndexShard#updateRoutingEntry(ShardRouting)}, * if this shadow replica needs to be promoted to a primary, the shard is * failed in order to allow a new primary to be re-allocated. */ @Override - public void updateRoutingEntry(ShardRouting newRouting, boolean persistState) throws IOException { + public void updateRoutingEntry(ShardRouting newRouting) throws IOException { if (newRouting.primary() == true) {// becoming a primary throw new IllegalStateException("can't promote shard to primary"); } - super.updateRoutingEntry(newRouting, persistState); + super.updateRoutingEntry(newRouting); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 62173f936c5..dbfcad6048a 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -131,16 +131,7 @@ final class StoreRecovery { } final void addIndices(RecoveryState.Index indexRecoveryStats, Directory target, Directory... sources) throws IOException { - /* - * TODO: once we upgraded to Lucene 6.1 use HardlinkCopyDirectoryWrapper to enable hardlinks if possible and enable it - * in the security.policy: - * - * grant codeBase "${codebase.lucene-misc-6.1.0.jar}" { - * // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper - * permission java.nio.file.LinkPermission "hard"; - * }; - * target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); - */ + target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(target, indexRecoveryStats), new IndexWriterConfig(null) .setCommitOnClose(false) diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java index e0032fe503b..94337ecdbc5 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -49,7 +51,6 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.iterable.Iterables; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; @@ -458,7 +459,9 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements } if (latest >= 0) { try { - return new Tuple<>(indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest)), latest); + final BlobStoreIndexShardSnapshots shardSnapshots = + indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest)); + return new Tuple<>(shardSnapshots, latest); } catch (IOException e) { logger.warn("failed to read index file [{}]", e, SNAPSHOT_INDEX_PREFIX + latest); } @@ -503,10 +506,8 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements */ public SnapshotContext(SnapshotId snapshotId, ShardId shardId, IndexShardSnapshotStatus snapshotStatus) { super(snapshotId, Version.CURRENT, shardId); - IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - store = indexService.getShardOrNull(shardId.id()).store(); this.snapshotStatus = snapshotStatus; - + store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store(); } /** @@ -788,8 +789,8 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements */ public RestoreContext(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) { super(snapshotId, version, shardId, snapshotShardId); - store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store(); this.recoveryState = recoveryState; + store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store(); } /** @@ -800,6 +801,25 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements try { logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId); BlobStoreIndexShardSnapshot snapshot = loadSnapshot(); + + if (snapshot.indexFiles().size() == 1 + && snapshot.indexFiles().get(0).physicalName().startsWith("segments_") + && snapshot.indexFiles().get(0).hasUnknownChecksum()) { + // If the shard has no documents, it will only contain a single segments_N file for the + // shard's snapshot. If we are restoring a snapshot created by a previous supported version, + // it is still possible that in that version, an empty shard has a segments_N file with an unsupported + // version (and no checksum), because we don't know the Lucene version to assign segments_N until we + // have written some data. Since the segments_N for an empty shard could have an incompatible Lucene + // version number and no checksum, even though the index itself is perfectly fine to restore, this + // empty shard would cause exceptions to be thrown. Since there is no data to restore from an empty + // shard anyway, we just create the empty shard here and then exit. + IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(null) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE) + .setCommitOnClose(true)); + writer.close(); + return; + } + SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()); final Store.MetadataSnapshot recoveryTargetMetadata; try { diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 60b7ec2112e..5bb0f728bc1 100644 --- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.snapshots.blobstore; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; @@ -50,6 +49,8 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil * Information about snapshotted file */ public static class FileInfo { + private static final String UNKNOWN_CHECKSUM = "_na_"; + private final String name; private final ByteSizeValue partSize; private final long partBytes; @@ -207,27 +208,43 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil * @return true if file in a store this this file have the same checksum and length */ public boolean isSame(FileInfo fileInfo) { - if (numberOfParts != fileInfo.numberOfParts) return false; - if (partBytes != fileInfo.partBytes) return false; - if (!name.equals(fileInfo.name)) return false; + if (numberOfParts != fileInfo.numberOfParts) { + return false; + } + if (partBytes != fileInfo.partBytes) { + return false; + } + if (!name.equals(fileInfo.name)) { + return false; + } if (partSize != null) { - if (!partSize.equals(fileInfo.partSize)) return false; + if (!partSize.equals(fileInfo.partSize)) { + return false; + } } else { - if (fileInfo.partSize != null) return false; + if (fileInfo.partSize != null) { + return false; + } } return metadata.isSame(fileInfo.metadata); } - static final class Fields { - static final String NAME = "name"; - static final String PHYSICAL_NAME = "physical_name"; - static final String LENGTH = "length"; - static final String CHECKSUM = "checksum"; - static final String PART_SIZE = "part_size"; - static final String WRITTEN_BY = "written_by"; - static final String META_HASH = "meta_hash"; + /** + * Checks if the checksum for the file is unknown. This only is possible on an empty shard's + * segments_N file which was created in older Lucene versions. + */ + public boolean hasUnknownChecksum() { + return metadata.checksum().equals(UNKNOWN_CHECKSUM); } + static final String NAME = "name"; + static final String PHYSICAL_NAME = "physical_name"; + static final String LENGTH = "length"; + static final String CHECKSUM = "checksum"; + static final String PART_SIZE = "part_size"; + static final String WRITTEN_BY = "written_by"; + static final String META_HASH = "meta_hash"; + /** * Serializes file info into JSON * @@ -237,22 +254,22 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil */ public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); - builder.field(Fields.NAME, file.name); - builder.field(Fields.PHYSICAL_NAME, file.metadata.name()); - builder.field(Fields.LENGTH, file.metadata.length()); - if (file.metadata.checksum() != null) { - builder.field(Fields.CHECKSUM, file.metadata.checksum()); + builder.field(NAME, file.name); + builder.field(PHYSICAL_NAME, file.metadata.name()); + builder.field(LENGTH, file.metadata.length()); + if (file.metadata.checksum().equals(UNKNOWN_CHECKSUM) == false) { + builder.field(CHECKSUM, file.metadata.checksum()); } if (file.partSize != null) { - builder.field(Fields.PART_SIZE, file.partSize.bytes()); + builder.field(PART_SIZE, file.partSize.bytes()); } if (file.metadata.writtenBy() != null) { - builder.field(Fields.WRITTEN_BY, file.metadata.writtenBy()); + builder.field(WRITTEN_BY, file.metadata.writtenBy()); } if (file.metadata.hash() != null && file.metadata().hash().length > 0) { - builder.field(Fields.META_HASH, file.metadata.hash()); + builder.field(META_HASH, file.metadata.hash()); } builder.endObject(); } @@ -271,6 +288,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil String checksum = null; ByteSizeValue partSize = null; Version writtenBy = null; + String writtenByStr = null; BytesRef metaHash = new BytesRef(); if (token == XContentParser.Token.START_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -278,19 +296,20 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil String currentFieldName = parser.currentName(); token = parser.nextToken(); if (token.isValue()) { - if ("name".equals(currentFieldName)) { + if (NAME.equals(currentFieldName)) { name = parser.text(); - } else if ("physical_name".equals(currentFieldName)) { + } else if (PHYSICAL_NAME.equals(currentFieldName)) { physicalName = parser.text(); - } else if ("length".equals(currentFieldName)) { + } else if (LENGTH.equals(currentFieldName)) { length = parser.longValue(); - } else if ("checksum".equals(currentFieldName)) { + } else if (CHECKSUM.equals(currentFieldName)) { checksum = parser.text(); - } else if ("part_size".equals(currentFieldName)) { + } else if (PART_SIZE.equals(currentFieldName)) { partSize = new ByteSizeValue(parser.longValue()); - } else if ("written_by".equals(currentFieldName)) { - writtenBy = Lucene.parseVersionLenient(parser.text(), null); - } else if ("meta_hash".equals(currentFieldName)) { + } else if (WRITTEN_BY.equals(currentFieldName)) { + writtenByStr = parser.text(); + writtenBy = Lucene.parseVersionLenient(writtenByStr, null); + } else if (META_HASH.equals(currentFieldName)) { metaHash.bytes = parser.binaryValue(); metaHash.offset = 0; metaHash.length = metaHash.bytes.length; @@ -305,6 +324,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil } } } + // Verify that file information is complete if (name == null || Strings.validFileName(name) == false) { throw new ElasticsearchParseException("missing or invalid file name [" + name + "]"); @@ -312,10 +332,29 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil throw new ElasticsearchParseException("missing or invalid physical file name [" + physicalName + "]"); } else if (length < 0) { throw new ElasticsearchParseException("missing or invalid file length"); + } else if (writtenBy == null) { + throw new ElasticsearchParseException("missing or invalid written_by [" + writtenByStr + "]"); + } else if (checksum == null) { + if (physicalName.startsWith("segments_") + && writtenBy.onOrAfter(StoreFileMetaData.FIRST_LUCENE_CHECKSUM_VERSION) == false) { + // its possible the checksum is null for segments_N files that belong to a shard with no data, + // so we will assign it _na_ for now and try to get the checksum from the file itself later + checksum = UNKNOWN_CHECKSUM; + } else { + throw new ElasticsearchParseException("missing checksum for name [" + name + "]"); + } } return new FileInfo(name, new StoreFileMetaData(physicalName, length, checksum, writtenBy, metaHash), partSize); } + @Override + public String toString() { + return "[name: " + name + + ", numberOfParts: " + numberOfParts + + ", partSize: " + partSize + + ", partBytes: " + partBytes + + ", metadata: " + metadata + "]"; + } } private final String snapshot; @@ -424,26 +463,21 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil return totalSize; } - static final class Fields { - static final String NAME = "name"; - static final String INDEX_VERSION = "index_version"; - static final String START_TIME = "start_time"; - static final String TIME = "time"; - static final String NUMBER_OF_FILES = "number_of_files"; - static final String TOTAL_SIZE = "total_size"; - static final String FILES = "files"; - } - - static final class ParseFields { - static final ParseField NAME = new ParseField("name"); - static final ParseField INDEX_VERSION = new ParseField("index_version", "index-version"); - static final ParseField START_TIME = new ParseField("start_time"); - static final ParseField TIME = new ParseField("time"); - static final ParseField NUMBER_OF_FILES = new ParseField("number_of_files"); - static final ParseField TOTAL_SIZE = new ParseField("total_size"); - static final ParseField FILES = new ParseField("files"); - } + private static final String NAME = "name"; + private static final String INDEX_VERSION = "index_version"; + private static final String START_TIME = "start_time"; + private static final String TIME = "time"; + private static final String NUMBER_OF_FILES = "number_of_files"; + private static final String TOTAL_SIZE = "total_size"; + private static final String FILES = "files"; + private static final ParseField PARSE_NAME = new ParseField("name"); + private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version"); + private static final ParseField PARSE_START_TIME = new ParseField("start_time"); + private static final ParseField PARSE_TIME = new ParseField("time"); + private static final ParseField PARSE_NUMBER_OF_FILES = new ParseField("number_of_files"); + private static final ParseField PARSE_TOTAL_SIZE = new ParseField("total_size"); + private static final ParseField PARSE_FILES = new ParseField("files"); /** * Serializes shard snapshot metadata info into JSON @@ -453,13 +487,13 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.NAME, snapshot); - builder.field(Fields.INDEX_VERSION, indexVersion); - builder.field(Fields.START_TIME, startTime); - builder.field(Fields.TIME, time); - builder.field(Fields.NUMBER_OF_FILES, numberOfFiles); - builder.field(Fields.TOTAL_SIZE, totalSize); - builder.startArray(Fields.FILES); + builder.field(NAME, snapshot); + builder.field(INDEX_VERSION, indexVersion); + builder.field(START_TIME, startTime); + builder.field(TIME, time); + builder.field(NUMBER_OF_FILES, numberOfFiles); + builder.field(TOTAL_SIZE, totalSize); + builder.startArray(FILES); for (FileInfo fileInfo : indexFiles) { FileInfo.toXContent(fileInfo, builder, params); } @@ -493,24 +527,24 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil String currentFieldName = parser.currentName(); token = parser.nextToken(); if (token.isValue()) { - if (parseFieldMatcher.match(currentFieldName, ParseFields.NAME)) { + if (parseFieldMatcher.match(currentFieldName, PARSE_NAME)) { snapshot = parser.text(); - } else if (parseFieldMatcher.match(currentFieldName, ParseFields.INDEX_VERSION)) { + } else if (parseFieldMatcher.match(currentFieldName, PARSE_INDEX_VERSION)) { // The index-version is needed for backward compatibility with v 1.0 indexVersion = parser.longValue(); - } else if (parseFieldMatcher.match(currentFieldName, ParseFields.START_TIME)) { + } else if (parseFieldMatcher.match(currentFieldName, PARSE_START_TIME)) { startTime = parser.longValue(); - } else if (parseFieldMatcher.match(currentFieldName, ParseFields.TIME)) { + } else if (parseFieldMatcher.match(currentFieldName, PARSE_TIME)) { time = parser.longValue(); - } else if (parseFieldMatcher.match(currentFieldName, ParseFields.NUMBER_OF_FILES)) { + } else if (parseFieldMatcher.match(currentFieldName, PARSE_NUMBER_OF_FILES)) { numberOfFiles = parser.intValue(); - } else if (parseFieldMatcher.match(currentFieldName, ParseFields.TOTAL_SIZE)) { + } else if (parseFieldMatcher.match(currentFieldName, PARSE_TOTAL_SIZE)) { totalSize = parser.longValue(); } else { throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); } } else if (token == XContentParser.Token.START_ARRAY) { - if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES)) { + if (parseFieldMatcher.match(currentFieldName, PARSE_FILES)) { while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { indexFiles.add(FileInfo.fromXContent(parser)); } @@ -526,6 +560,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil } } return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles), - startTime, time, numberOfFiles, totalSize); + startTime, time, numberOfFiles, totalSize); } + } diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java index a720f5cb258..166f978a4db 100644 --- a/core/src/main/java/org/elasticsearch/index/store/Store.java +++ b/core/src/main/java/org/elasticsearch/index/store/Store.java @@ -41,7 +41,6 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.SimpleFSDirectory; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; @@ -444,11 +443,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref } /** - * The returned IndexOutput might validate the files checksum if the file has been written with a newer lucene version - * and the metadata holds the necessary information to detect that it was been written by Lucene 4.8 or newer. If it has only - * a legacy checksum, returned IndexOutput will not verify the checksum. + * The returned IndexOutput validates the files checksum. *

- * Note: Checksums are calculated nevertheless since lucene does it by default sicne version 4.8.0. This method only adds the + * Note: Checksums are calculated by default since version 4.8.0. This method only adds the * verification against the checksum in the given metadata and does not add any significant overhead. */ public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException { @@ -652,17 +649,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref // different in the diff. That's why we have to double check here again if the rest of it matches. // all is fine this file is just part of a commit or a segment that is different - final boolean same = local.isSame(remote); - - // this check ensures that the two files are consistent ie. if we don't have checksums only the rest needs to match we are just - // verifying that we are consistent on both ends source and target - final boolean hashAndLengthEqual = ( - local.checksum() == null - && remote.checksum() == null - && local.hash().equals(remote.hash()) - && local.length() == remote.length()); - final boolean consistent = hashAndLengthEqual || same; - if (consistent == false) { + if (local.isSame(remote) == false) { logger.debug("Files are different on the recovery target: {} ", recoveryDiff); throw new IllegalStateException("local version: " + local + " is different from remote version after recovery: " + remote, null); } @@ -898,18 +885,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref } } - /** - * Computes a strong hash value for small files. Note that this method should only be used for files < 1MB - */ - public static BytesRef hashFile(Directory directory, String file) throws IOException { - final BytesRefBuilder fileHash = new BytesRefBuilder(); - try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) { - hashFile(fileHash, new InputStreamIndexInput(in, in.length()), in.length()); - } - return fileHash.get(); - } - - /** * Computes a strong hash value for small files. Note that this method should only be used for files < 1MB */ diff --git a/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java b/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java index e163b15f60e..2653f01c81d 100644 --- a/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java +++ b/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java @@ -21,10 +21,8 @@ package org.elasticsearch.index.store; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; @@ -58,14 +56,15 @@ public class StoreFileMetaData implements Writeable { } public StoreFileMetaData(String name, long length, String checksum, Version writtenBy, BytesRef hash) { - assert writtenBy != null && writtenBy.onOrAfter(FIRST_LUCENE_CHECKSUM_VERSION) : "index version less that " - + FIRST_LUCENE_CHECKSUM_VERSION + " are not supported but got: " + writtenBy; - Objects.requireNonNull(writtenBy, "writtenBy must not be null"); - Objects.requireNonNull(checksum, "checksum must not be null"); - this.name = name; + // its possible here to have a _na_ checksum or an unsupported writtenBy version, if the + // file is a segments_N file, but that is fine in the case of a segments_N file because + // we handle that case upstream + assert name.startsWith("segments_") || (writtenBy != null && writtenBy.onOrAfter(FIRST_LUCENE_CHECKSUM_VERSION)) : + "index version less that " + FIRST_LUCENE_CHECKSUM_VERSION + " are not supported but got: " + writtenBy; + this.name = Objects.requireNonNull(name, "name must not be null"); this.length = length; - this.checksum = checksum; - this.writtenBy = writtenBy; + this.checksum = Objects.requireNonNull(checksum, "checksum must not be null"); + this.writtenBy = Objects.requireNonNull(writtenBy, "writtenBy must not be null"); this.hash = hash == null ? new BytesRef() : hash; } diff --git a/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java new file mode 100644 index 00000000000..c0d929d82f5 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.cache.RemovalNotification; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.cache.request.ShardRequestCache; +import org.elasticsearch.index.shard.IndexShard; + +import java.io.IOException; + +/** + * Abstract base class for the an {@link IndexShard} level {@linkplain IndicesRequestCache.CacheEntity}. + */ +abstract class AbstractIndexShardCacheEntity implements IndicesRequestCache.CacheEntity { + @FunctionalInterface + public interface Loader { + void load(StreamOutput out) throws IOException; + } + + private final Loader loader; + private boolean loadedFromCache = true; + + protected AbstractIndexShardCacheEntity(Loader loader) { + this.loader = loader; + } + + /** + * When called after passing this through + * {@link IndicesRequestCache#getOrCompute(IndicesRequestCache.CacheEntity, DirectoryReader, BytesReference)} this will return whether + * or not the result was loaded from the cache. + */ + public final boolean loadedFromCache() { + return loadedFromCache; + } + + /** + * Get the {@linkplain ShardRequestCache} used to track cache statistics. + */ + protected abstract ShardRequestCache stats(); + + @Override + public final IndicesRequestCache.Value loadValue() throws IOException { + /* BytesStreamOutput allows to pass the expected size but by default uses + * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie. + * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful + * since we don't shrink to the actual size once we are done serializing. + * By passing 512 as the expected size we will resize the byte array in the stream + * slowly until we hit the page size and don't waste too much memory for small query + * results.*/ + final int expectedSizeInBytes = 512; + try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) { + loader.load(out); + // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep + // the memory properly paged instead of having varied sized bytes + final BytesReference reference = out.bytes(); + loadedFromCache = false; + return new IndicesRequestCache.Value(reference, out.ramBytesUsed()); + } + } + + @Override + public final void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) { + stats().onCached(key, value); + } + + @Override + public final void onHit() { + stats().onHit(); + } + + @Override + public final void onMiss() { + stats().onMiss(); + } + + @Override + public final void onRemoval(RemovalNotification notification) { + stats().onRemoval(notification.getKey(), notification.getValue(), + notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED); + } +} diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index bd01e7f0183..70b9443e043 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -45,6 +45,7 @@ import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable { @@ -52,6 +53,9 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, "indices.queries.cache.size", "10%", Property.NodeScope); public static final Setting INDICES_CACHE_QUERY_COUNT_SETTING = Setting.intSetting( "indices.queries.cache.count", 10000, 1, Property.NodeScope); + // enables caching on all segments instead of only the larger ones, for testing only + public static final Setting INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = Setting.boolSetting( + "indices.queries.cache.all_segments", false, Property.NodeScope); private final LRUQueryCache cache; private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap(); @@ -69,111 +73,11 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, final int count = INDICES_CACHE_QUERY_COUNT_SETTING.get(settings); logger.debug("using [node] query cache with size [{}] max filter count [{}]", size, count); - cache = new LRUQueryCache(count, size.bytes()) { - - private Stats getStats(Object coreKey) { - final ShardId shardId = shardKeyMap.getShardId(coreKey); - if (shardId == null) { - return null; - } - return shardStats.get(shardId); - } - - private Stats getOrCreateStats(Object coreKey) { - final ShardId shardId = shardKeyMap.getShardId(coreKey); - Stats stats = shardStats.get(shardId); - if (stats == null) { - stats = new Stats(); - shardStats.put(shardId, stats); - } - return stats; - } - - // It's ok to not protect these callbacks by a lock since it is - // done in LRUQueryCache - @Override - protected void onClear() { - assert Thread.holdsLock(this); - super.onClear(); - for (Stats stats : shardStats.values()) { - // don't throw away hit/miss - stats.cacheSize = 0; - stats.ramBytesUsed = 0; - } - sharedRamBytesUsed = 0; - } - - @Override - protected void onQueryCache(Query filter, long ramBytesUsed) { - assert Thread.holdsLock(this); - super.onQueryCache(filter, ramBytesUsed); - sharedRamBytesUsed += ramBytesUsed; - } - - @Override - protected void onQueryEviction(Query filter, long ramBytesUsed) { - assert Thread.holdsLock(this); - super.onQueryEviction(filter, ramBytesUsed); - sharedRamBytesUsed -= ramBytesUsed; - } - - @Override - protected void onDocIdSetCache(Object readerCoreKey, long ramBytesUsed) { - assert Thread.holdsLock(this); - super.onDocIdSetCache(readerCoreKey, ramBytesUsed); - final Stats shardStats = getOrCreateStats(readerCoreKey); - shardStats.cacheSize += 1; - shardStats.cacheCount += 1; - shardStats.ramBytesUsed += ramBytesUsed; - - StatsAndCount statsAndCount = stats2.get(readerCoreKey); - if (statsAndCount == null) { - statsAndCount = new StatsAndCount(shardStats); - stats2.put(readerCoreKey, statsAndCount); - } - statsAndCount.count += 1; - } - - @Override - protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sumRamBytesUsed) { - assert Thread.holdsLock(this); - super.onDocIdSetEviction(readerCoreKey, numEntries, sumRamBytesUsed); - // onDocIdSetEviction might sometimes be called with a number - // of entries equal to zero if the cache for the given segment - // was already empty when the close listener was called - if (numEntries > 0) { - // We can't use ShardCoreKeyMap here because its core closed - // listener is called before the listener of the cache which - // triggers this eviction. So instead we use use stats2 that - // we only evict when nothing is cached anymore on the segment - // instead of relying on close listeners - final StatsAndCount statsAndCount = stats2.get(readerCoreKey); - final Stats shardStats = statsAndCount.stats; - shardStats.cacheSize -= numEntries; - shardStats.ramBytesUsed -= sumRamBytesUsed; - statsAndCount.count -= numEntries; - if (statsAndCount.count == 0) { - stats2.remove(readerCoreKey); - } - } - } - - @Override - protected void onHit(Object readerCoreKey, Query filter) { - assert Thread.holdsLock(this); - super.onHit(readerCoreKey, filter); - final Stats shardStats = getStats(readerCoreKey); - shardStats.hitCount += 1; - } - - @Override - protected void onMiss(Object readerCoreKey, Query filter) { - assert Thread.holdsLock(this); - super.onMiss(readerCoreKey, filter); - final Stats shardStats = getOrCreateStats(readerCoreKey); - shardStats.missCount += 1; - } - }; + if (INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.get(settings)) { + cache = new ElasticsearchLRUQueryCache(count, size.bytes(), context -> true); + } else { + cache = new ElasticsearchLRUQueryCache(count, size.bytes()); + } sharedRamBytesUsed = 0; } @@ -316,4 +220,111 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache, assert empty(shardStats.get(shardId)); shardStats.remove(shardId); } + + private class ElasticsearchLRUQueryCache extends LRUQueryCache { + + ElasticsearchLRUQueryCache(int maxSize, long maxRamBytesUsed, Predicate leavesToCache) { + super(maxSize, maxRamBytesUsed, leavesToCache); + } + + ElasticsearchLRUQueryCache(int maxSize, long maxRamBytesUsed) { + super(maxSize, maxRamBytesUsed); + } + + private Stats getStats(Object coreKey) { + final ShardId shardId = shardKeyMap.getShardId(coreKey); + if (shardId == null) { + return null; + } + return shardStats.get(shardId); + } + + private Stats getOrCreateStats(Object coreKey) { + final ShardId shardId = shardKeyMap.getShardId(coreKey); + Stats stats = shardStats.get(shardId); + if (stats == null) { + stats = new Stats(); + shardStats.put(shardId, stats); + } + return stats; + } + + // It's ok to not protect these callbacks by a lock since it is + // done in LRUQueryCache + @Override + protected void onClear() { + super.onClear(); + for (Stats stats : shardStats.values()) { + // don't throw away hit/miss + stats.cacheSize = 0; + stats.ramBytesUsed = 0; + } + sharedRamBytesUsed = 0; + } + + @Override + protected void onQueryCache(Query filter, long ramBytesUsed) { + super.onQueryCache(filter, ramBytesUsed); + sharedRamBytesUsed += ramBytesUsed; + } + + @Override + protected void onQueryEviction(Query filter, long ramBytesUsed) { + super.onQueryEviction(filter, ramBytesUsed); + sharedRamBytesUsed -= ramBytesUsed; + } + + @Override + protected void onDocIdSetCache(Object readerCoreKey, long ramBytesUsed) { + super.onDocIdSetCache(readerCoreKey, ramBytesUsed); + final Stats shardStats = getOrCreateStats(readerCoreKey); + shardStats.cacheSize += 1; + shardStats.cacheCount += 1; + shardStats.ramBytesUsed += ramBytesUsed; + + StatsAndCount statsAndCount = stats2.get(readerCoreKey); + if (statsAndCount == null) { + statsAndCount = new StatsAndCount(shardStats); + stats2.put(readerCoreKey, statsAndCount); + } + statsAndCount.count += 1; + } + + @Override + protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sumRamBytesUsed) { + super.onDocIdSetEviction(readerCoreKey, numEntries, sumRamBytesUsed); + // onDocIdSetEviction might sometimes be called with a number + // of entries equal to zero if the cache for the given segment + // was already empty when the close listener was called + if (numEntries > 0) { + // We can't use ShardCoreKeyMap here because its core closed + // listener is called before the listener of the cache which + // triggers this eviction. So instead we use use stats2 that + // we only evict when nothing is cached anymore on the segment + // instead of relying on close listeners + final StatsAndCount statsAndCount = stats2.get(readerCoreKey); + final Stats shardStats = statsAndCount.stats; + shardStats.cacheSize -= numEntries; + shardStats.ramBytesUsed -= sumRamBytesUsed; + statsAndCount.count -= numEntries; + if (statsAndCount.count == 0) { + stats2.remove(readerCoreKey); + } + } + } + + @Override + protected void onHit(Object readerCoreKey, Query filter) { + super.onHit(readerCoreKey, filter); + final Stats shardStats = getStats(readerCoreKey); + shardStats.hitCount += 1; + } + + @Override + protected void onMiss(Object readerCoreKey, Query filter) { + super.onMiss(readerCoreKey, filter); + final Stats shardStats = getOrCreateStats(readerCoreKey); + shardStats.missCount += 1; + } + } } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index ba512379868..92bcfbe64a2 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -20,6 +20,7 @@ package org.elasticsearch.indices; import com.carrotsearch.hppc.cursors.ObjectCursor; + import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.CollectionUtil; @@ -28,6 +29,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; +import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchType; @@ -35,15 +37,15 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.cache.RemovalNotification; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -55,6 +57,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.Callback; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.env.NodeEnvironment; @@ -69,9 +72,11 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.NodeServicesProvider; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.cache.request.ShardRequestCache; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.recovery.RecoveryStats; @@ -85,11 +90,16 @@ import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.AbstractIndexShardCacheEntity.Loader; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.RecoveryTargetService; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.query.QueryPhase; @@ -124,7 +134,8 @@ import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; /** * */ -public class IndicesService extends AbstractLifecycleComponent implements Iterable, IndexService.ShardStoreDeleter { +public class IndicesService extends AbstractLifecycleComponent + implements IndicesClusterStateService.AllocatedIndices, IndexService.ShardStoreDeleter { public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout"; public static final Setting INDICES_CACHE_CLEAN_INTERVAL_SETTING = @@ -296,11 +307,14 @@ public class IndicesService extends AbstractLifecycleComponent i } /** - * Returns true if changes (adding / removing) indices, shards and so on are allowed. + * Checks if changes (adding / removing) indices, shards and so on are allowed. + * + * @throws IllegalStateException if no changes allowed. */ - public boolean changesAllowed() { - // we check on stop here since we defined stop when we delete the indices - return lifecycle.started(); + private void ensureChangesAllowed() { + if (lifecycle.started() == false) { + throw new IllegalStateException("Can't make changes to indices service, node is closed"); + } } @Override @@ -314,10 +328,9 @@ public class IndicesService extends AbstractLifecycleComponent i /** * Returns an IndexService for the specified index if exists otherwise returns null. - * */ - @Nullable - public IndexService indexService(Index index) { + @Override + public @Nullable IndexService indexService(Index index) { return indices.get(index.getUUID()); } @@ -339,11 +352,9 @@ public class IndicesService extends AbstractLifecycleComponent i * @param builtInListeners a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with the per-index listeners * @throws IndexAlreadyExistsException if the index already exists. */ + @Override public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List builtInListeners) throws IOException { - - if (!lifecycle.started()) { - throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed"); - } + ensureChangesAllowed(); if (indexMetaData.getIndexUUID().equals(IndexMetaData.INDEX_UUID_NA_VALUE)) { throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]"); } @@ -424,14 +435,44 @@ public class IndicesService extends AbstractLifecycleComponent i } } + @Override + public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService, + RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, + NodeServicesProvider nodeServicesProvider, Callback onShardFailure) throws IOException { + ensureChangesAllowed(); + IndexService indexService = indexService(shardRouting.index()); + IndexShard indexShard = indexService.createShard(shardRouting); + indexShard.addShardFailureCallback(onShardFailure); + indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService, + (type, mapping) -> { + assert recoveryState.getType() == RecoveryState.Type.LOCAL_SHARDS : + "mapping update consumer only required by local shards recovery"; + try { + nodeServicesProvider.getClient().admin().indices().preparePutMapping() + .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid + .setType(type) + .setSource(mapping.source().string()) + .get(); + } catch (IOException ex) { + throw new ElasticsearchException("failed to stringify mapping source", ex); + } + }, this); + return indexShard; + } + /** * Removes the given index from this service and releases all associated resources. Persistent parts of the index * like the shards files, state and transaction logs are kept around in the case of a disaster recovery. * @param index the index to remove * @param reason the high level reason causing this removal */ + @Override public void removeIndex(Index index, String reason) { - removeIndex(index, reason, false); + try { + removeIndex(index, reason, false); + } catch (Throwable e) { + logger.warn("failed to remove index ({})", e, reason); + } } private void removeIndex(Index index, String reason, boolean delete) { @@ -516,14 +557,20 @@ public class IndicesService extends AbstractLifecycleComponent i * @param index the index to delete * @param reason the high level reason causing this delete */ - public void deleteIndex(Index index, String reason) throws IOException { - removeIndex(index, reason, true); + @Override + public void deleteIndex(Index index, String reason) { + try { + removeIndex(index, reason, true); + } catch (Throwable e) { + logger.warn("failed to delete index ({})", e, reason); + } } /** * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)} */ + @Override public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex().getName(); @@ -683,8 +730,8 @@ public class IndicesService extends AbstractLifecycleComponent i * @param clusterState {@code ClusterState} to ensure the index is not part of it * @return IndexMetaData for the index loaded from disk */ - @Nullable - public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) { + @Override + public @Nullable IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) { // this method should only be called when we know the index (name + uuid) is not part of the cluster state if (clusterState.metaData().index(index) != null) { throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state."); @@ -839,6 +886,7 @@ public class IndicesService extends AbstractLifecycleComponent i * @param index the index to process the pending deletes for * @param timeout the timeout used for processing pending deletes */ + @Override public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeout) throws IOException, InterruptedException { logger.debug("{} processing pending deletes", index); final long startTimeNS = System.nanoTime(); @@ -1041,9 +1089,10 @@ public class IndicesService extends AbstractLifecycleComponent i if (shard == null) { return; } - indicesRequestCache.clear(new IndexShardCacheEntity(shard)); + indicesRequestCache.clear(new IndexShardCacheEntity(shard, null)); logger.trace("{} explicit cache clear", shard.shardId()); } + /** * Loads the cache result, computing it if needed by executing the query phase and otherwise deserializing the cached * value into the {@link SearchContext#queryResult() context's query result}. The combination of load + compute allows @@ -1052,10 +1101,13 @@ public class IndicesService extends AbstractLifecycleComponent i */ public void loadIntoContext(ShardSearchRequest request, SearchContext context, QueryPhase queryPhase) throws Exception { assert canCache(request, context); - final IndexShardCacheEntity entity = new IndexShardCacheEntity(context.indexShard(), queryPhase, context); + final IndexShardCacheEntity entity = new IndexShardCacheEntity(context.indexShard(), out -> { + queryPhase.execute(context); + context.queryResult().writeToNoId(out); + }); final DirectoryReader directoryReader = context.searcher().getDirectoryReader(); final BytesReference bytesReference = indicesRequestCache.getOrCompute(entity, directoryReader, request.cacheKey()); - if (entity.loaded == false) { // if we have loaded this we don't need to do anything + if (entity.loadedFromCache()) { // restore the cached query result into the context final QuerySearchResult result = context.queryResult(); StreamInput in = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), namedWriteableRegistry); @@ -1064,51 +1116,57 @@ public class IndicesService extends AbstractLifecycleComponent i } } - static final class IndexShardCacheEntity implements IndicesRequestCache.CacheEntity { - private final QueryPhase queryPhase; - private final SearchContext context; + /** + * Fetch {@linkplain FieldStats} for a field. These stats are cached until the shard changes. + * @param shard the shard to use with the cache key + * @param searcher searcher to use to lookup the field stats + * @param field the actual field + * @param useCache should this request use the cache? + */ + public FieldStats getFieldStats(IndexShard shard, Engine.Searcher searcher, String field, boolean useCache) throws Exception { + MappedFieldType fieldType = shard.mapperService().fullName(field); + if (fieldType == null) { + return null; + } + if (useCache == false) { + return fieldType.stats(searcher.reader()); + } + BytesReference cacheKey = new BytesArray("fieldstats:" + field); + BytesReference statsRef = cacheShardLevelResult(shard, searcher.getDirectoryReader(), cacheKey, out -> { + out.writeOptionalWriteable(fieldType.stats(searcher.reader())); + }); + try (StreamInput in = StreamInput.wrap(statsRef)) { + return in.readOptionalWriteable(FieldStats::readFrom); + } + } + + /** + * Cache something calculated at the shard level. + * @param shard the shard this item is part of + * @param reader a reader for this shard. Used to invalidate the cache when there are changes. + * @param cacheKey key for the thing being cached within this shard + * @param loader loads the data into the cache if needed + * @return the contents of the cache or the result of calling the loader + */ + private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, Loader loader) + throws Exception { + IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard, loader); + return indicesRequestCache.getOrCompute(cacheEntity, reader, cacheKey); + } + + final static class IndexShardCacheEntity extends AbstractIndexShardCacheEntity { private final IndexShard indexShard; - private final ShardRequestCache requestCache; - private boolean loaded = false; - IndexShardCacheEntity(IndexShard indexShard) { - this(indexShard, null, null); - } - - public IndexShardCacheEntity(IndexShard indexShard, QueryPhase queryPhase, SearchContext context) { - this.queryPhase = queryPhase; - this.context = context; + protected IndexShardCacheEntity(IndexShard indexShard, Loader loader) { + super(loader); this.indexShard = indexShard; - this.requestCache = indexShard.requestCache(); } @Override - public IndicesRequestCache.Value loadValue() throws IOException { - queryPhase.execute(context); - /* BytesStreamOutput allows to pass the expected size but by default uses - * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie. - * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful - * since we don't shrink to the actual size once we are done serializing. - * By passing 512 as the expected size we will resize the byte array in the stream - * slowly until we hit the page size and don't waste too much memory for small query - * results.*/ - final int expectedSizeInBytes = 512; - try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) { - context.queryResult().writeToNoId(out); - // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep - // the memory properly paged instead of having varied sized bytes - final BytesReference reference = out.bytes(); - loaded = true; - return new IndicesRequestCache.Value(reference, out.ramBytesUsed()); - } + protected ShardRequestCache stats() { + return indexShard.requestCache(); } - @Override - public void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) { - requestCache.onCached(key, value); - } - - @Override public boolean isOpen() { return indexShard.state() != IndexShardState.CLOSED; @@ -1118,22 +1176,6 @@ public class IndicesService extends AbstractLifecycleComponent i public Object getCacheIdentity() { return indexShard; } - - @Override - public void onHit() { - requestCache.onHit(); - } - - @Override - public void onMiss() { - requestCache.onMiss(); - } - - @Override - public void onRemoval(RemovalNotification notification) { - requestCache.onRemoval(notification.getKey(), notification.getValue(), notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED); - } - } @FunctionalInterface diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index 2c77f863c47..c0bfedc47ca 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -19,16 +19,13 @@ package org.elasticsearch.indices.cluster; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.store.LockObtainFailedException; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingNode; @@ -37,7 +34,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; @@ -48,17 +44,18 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexComponent; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexShardAlreadyExistsException; import org.elasticsearch.index.NodeServicesProvider; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardRelocatedException; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardNotFoundException; +import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.flush.SyncedFlushService; import org.elasticsearch.indices.recovery.RecoveryFailedException; @@ -73,7 +70,6 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -82,14 +78,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** * */ public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateListener { - private final IndicesService indicesService; + final AllocatedIndices> indicesService; private final ClusterService clusterService; private final ThreadPool threadPool; private final RecoveryTargetService recoveryTargetService; @@ -102,11 +97,10 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent failedShards = ConcurrentCollections.newConcurrentMap(); + final ConcurrentMap failedShardsCache = ConcurrentCollections.newConcurrentMap(); private final RestoreService restoreService; private final RepositoriesService repositoriesService; - private final Object mutex = new Object(); private final FailedShardHandler failedShardHandler = new FailedShardHandler(); private final boolean sendRefreshMapping; @@ -120,6 +114,22 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent>) indicesService, + clusterService, threadPool, recoveryTargetService, shardStateAction, + nodeMappingRefreshAction, repositoriesService, restoreService, searchService, syncedFlushService, recoverySource, + nodeServicesProvider); + } + + // for tests + IndicesClusterStateService(Settings settings, + AllocatedIndices> indicesService, + ClusterService clusterService, + ThreadPool threadPool, RecoveryTargetService recoveryTargetService, + ShardStateAction shardStateAction, + NodeMappingRefreshAction nodeMappingRefreshAction, + RepositoriesService repositoriesService, RestoreService restoreService, + SearchService searchService, SyncedFlushService syncedFlushService, + RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider) { super(settings); this.buildInIndexListener = Arrays.asList(recoverySource, recoveryTargetService, searchService, syncedFlushService); this.indicesService = indicesService; @@ -149,87 +159,97 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent indexService : indicesService) { + indicesService.removeIndex(indexService.index(), "cleaning index (disabled block persistence)"); // also cleans shards } - - cleanFailedShards(event); - - // cleaning up indices that are completely deleted so we won't need to worry about them - // when checking for shards - applyDeletedIndices(event); - applyDeletedShards(event); - // call after deleted shards so indices with no shards will be cleaned - applyCleanedIndices(event); - // make sure that newly created shards use the latest meta data - applyIndexMetaData(event); - applyNewIndices(event); - // apply mappings also updates new indices. TODO: make new indices good to begin with - applyMappings(event); - applyNewOrUpdatedShards(event); - } - } - - private void cleanFailedShards(final ClusterChangedEvent event) { - RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId()); - if (routingNode == null) { - failedShards.clear(); return; } - for (Iterator> iterator = failedShards.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry entry = iterator.next(); - ShardRouting failedShardRouting = entry.getValue(); - ShardRouting matchedShardRouting = routingNode.getByShardId(failedShardRouting.shardId()); - if (matchedShardRouting == null || matchedShardRouting.isSameAllocation(failedShardRouting) == false) { + + updateFailedShardsCache(state); + + deleteIndices(event); // also deletes shards of deleted indices + + removeUnallocatedIndices(state); // also removes shards of removed indices + + failMissingShards(state); + + removeShards(state); + + updateIndices(event); // can also fail shards, but these are then guaranteed to be in failedShardsCache + + createIndices(state); + + createOrUpdateShards(state); + } + + /** + * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master. + * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node. + * Resends shard failures for shards that are still marked as allocated to this node but previously failed. + * + * @param state new cluster state + */ + private void updateFailedShardsCache(final ClusterState state) { + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + if (localRoutingNode == null) { + failedShardsCache.clear(); + return; + } + + DiscoveryNode masterNode = state.nodes().getMasterNode(); + + // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet + for (Iterator> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) { + ShardRouting failedShardRouting = iterator.next().getValue(); + ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId()); + if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) { iterator.remove(); + } else { + if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction? + String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure"; + logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message); + shardStateAction.shardFailed(matchedRouting, matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER); + } } } } - private void applyDeletedIndices(final ClusterChangedEvent event) { + /** + * Deletes indices (with shard data). + * + * @param event cluster change event + */ + private void deleteIndices(final ClusterChangedEvent event) { final ClusterState previousState = event.previousState(); - final String localNodeId = event.state().nodes().getLocalNodeId(); + final ClusterState state = event.state(); + final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; for (Index index : event.indicesDeleted()) { if (logger.isDebugEnabled()) { logger.debug("[{}] cleaning index, no longer part of the metadata", index); } - final IndexService idxService = indicesService.indexService(index); + AllocatedIndex indexService = indicesService.indexService(index); final IndexSettings indexSettings; - if (idxService != null) { - indexSettings = idxService.getIndexSettings(); - deleteIndex(index, "index no longer part of the metadata"); + if (indexService != null) { + indexSettings = indexService.getIndexSettings(); + indicesService.deleteIndex(index, "index no longer part of the metadata"); } else if (previousState.metaData().hasIndex(index.getName())) { // The deleted index was part of the previous cluster state, but not loaded on the local node final IndexMetaData metaData = previousState.metaData().index(index); indexSettings = new IndexSettings(metaData, settings); - indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, event.state()); + indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, state); } else { // The previous cluster state's metadata also does not contain the index, // which is what happens on node startup when an index was deleted while the @@ -255,10 +275,10 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent indexService : indicesService) { + Index index = indexService.index(); + IndexMetaData indexMetaData = event.state().metaData().index(index); if (indexMetaData == null) { - assert false : "index" + indexService.index() + " exists locally, doesn't have a metadata but is not part " + assert false : "index" + index + " exists locally, doesn't have a metadata but is not part" + " of the delete index list. \nprevious state: " + event.previousState().prettyPrint() + "\n current state:\n" + event.state().prettyPrint(); - logger.warn("[{}] isn't part of metadata but is part of in memory structures. removing", - indexService.index()); - deleteIndex(indexService.index(), "isn't part of metadata (explicit check)"); + logger.warn("[{}] isn't part of metadata but is part of in memory structures. removing", index); + indicesService.deleteIndex(index, "isn't part of metadata (explicit check)"); } } } - private void applyDeletedShards(final ClusterChangedEvent event) { - RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId()); - if (routingNode == null) { + /** + * Removes indices that have no shards allocated to this node. This does not delete the shard data as we wait for enough + * shard copies to exist in the cluster before deleting shard data (triggered by {@link org.elasticsearch.indices.store.IndicesStore}). + * + * @param state new cluster state + */ + private void removeUnallocatedIndices(final ClusterState state) { + final String localNodeId = state.nodes().getLocalNodeId(); + assert localNodeId != null; + + Set indicesWithShards = new HashSet<>(); + RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); + if (localRoutingNode != null) { // null e.g. if we are not a data node + for (ShardRouting shardRouting : localRoutingNode) { + indicesWithShards.add(shardRouting.index()); + } + } + + for (AllocatedIndex indexService : indicesService) { + Index index = indexService.index(); + if (indicesWithShards.contains(index) == false) { + logger.debug("{} removing index, no shards allocated", index); + indicesService.removeIndex(index, "removing index (no shards allocated)"); + } + } + } + + /** + * Notifies master about shards that don't exist but are supposed to be active on this node. + * + * @param state new cluster state + */ + private void failMissingShards(final ClusterState state) { + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + if (localRoutingNode == null) { return; } - - final Map> shardsByIndex = new HashMap<>(); - for (ShardRouting shard : routingNode) { - shardsByIndex.computeIfAbsent(shard.index(), k -> new HashSet<>()).add(shard.allocationId().getId()); + for (final ShardRouting shardRouting : localRoutingNode) { + ShardId shardId = shardRouting.shardId(); + if (shardRouting.initializing() == false && + failedShardsCache.containsKey(shardId) == false && + indicesService.getShardOrNull(shardId) == null) { + // the master thinks we are active, but we don't have this shard at all, mark it as failed + sendFailShard(shardRouting, "master marked shard as active, but shard has not been created, mark shard as failed", null); + } } + } - for (IndexService indexService : indicesService) { - Index index = indexService.index(); - IndexMetaData indexMetaData = event.state().metaData().index(index); - assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index; - // now, go over and delete shards that needs to get deleted - Set newShardAllocationIds = shardsByIndex.getOrDefault(index, Collections.emptySet()); - for (IndexShard existingShard : indexService) { - if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) { - if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { - if (logger.isDebugEnabled()) { - logger.debug("{} removing shard (index is closed)", existingShard.shardId()); + /** + * Removes shards that are currently loaded by indicesService but have disappeared from the routing table of the current node. + * Also removes shards where the recovery source node has changed. + * This method does not delete the shard data. + * + * @param state new cluster state + */ + private void removeShards(final ClusterState state) { + final RoutingTable routingTable = state.routingTable(); + final DiscoveryNodes nodes = state.nodes(); + final String localNodeId = state.nodes().getLocalNodeId(); + assert localNodeId != null; + + // remove shards based on routing nodes (no deletion of data) + RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); + for (AllocatedIndex indexService : indicesService) { + for (Shard shard : indexService) { + ShardRouting currentRoutingEntry = shard.routingEntry(); + ShardId shardId = currentRoutingEntry.shardId(); + ShardRouting newShardRouting = localRoutingNode == null ? null : localRoutingNode.getByShardId(shardId); + if (newShardRouting == null || newShardRouting.isSameAllocation(currentRoutingEntry) == false) { + // we can just remove the shard without cleaning it locally, since we will clean it in IndicesStore + // once all shards are allocated + logger.debug("{} removing shard (not allocated)", shardId); + indexService.removeShard(shardId.id(), "removing shard (not allocated)"); + } else { + // remove shards where recovery source has changed. This re-initializes shards later in createOrUpdateShards + if (newShardRouting.isPeerRecovery()) { + RecoveryState recoveryState = shard.recoveryState(); + final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, newShardRouting); + if (recoveryState.getSourceNode().equals(sourceNode) == false) { + if (recoveryTargetService.cancelRecoveriesForShard(shardId, "recovery source node changed")) { + // getting here means that the shard was still recovering + logger.debug("{} removing shard (recovery source changed), current [{}], global [{}])", + shardId, currentRoutingEntry, newShardRouting); + indexService.removeShard(shardId.id(), "removing shard (recovery source node changed)"); + } } - indexService.removeShard(existingShard.shardId().id(), "removing shard (index is closed)"); - } else { - // we can just remove the shard, without cleaning it locally, since we will clean it - // when all shards are allocated in the IndicesStore - if (logger.isDebugEnabled()) { - logger.debug("{} removing shard (not allocated)", existingShard.shardId()); - } - indexService.removeShard(existingShard.shardId().id(), "removing shard (not allocated)"); } } } } } - private void applyCleanedIndices(final ClusterChangedEvent event) { - // handle closed indices, since they are not allocated on a node once they are closed - // so applyDeletedIndices might not take them into account - for (IndexService indexService : indicesService) { - Index index = indexService.index(); - IndexMetaData indexMetaData = event.state().metaData().index(index); - if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE) { - for (Integer shardId : indexService.shardIds()) { - logger.debug("{}[{}] removing shard (index is closed)", index, shardId); - try { - indexService.removeShard(shardId, "removing shard (index is closed)"); - } catch (Throwable e) { - logger.warn("{} failed to remove shard (index is closed)", e, index); - } - } - } - } - - final Set hasAllocations = new HashSet<>(); - final RoutingNode node = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId()); - // if no shards are allocated ie. if this node is a master-only node it can return nul - if (node != null) { - for (ShardRouting routing : node) { - hasAllocations.add(routing.index()); - } - } - for (IndexService indexService : indicesService) { - Index index = indexService.index(); - if (hasAllocations.contains(index) == false) { - assert indexService.shardIds().isEmpty() : - "no locally assigned shards, but index wasn't emptied by applyDeletedShards." - + " index " + index + ", shards: " + indexService.shardIds(); - if (logger.isDebugEnabled()) { - logger.debug("{} cleaning index (no shards allocated)", index); - } - // clean the index - removeIndex(index, "removing index (no shards allocated)"); - } - } - } - - private void applyIndexMetaData(ClusterChangedEvent event) { - if (!event.metaDataChanged()) { - return; - } - for (IndexMetaData indexMetaData : event.state().metaData()) { - if (!indicesService.hasIndex(indexMetaData.getIndex())) { - // we only create / update here - continue; - } - // if the index meta data didn't change, no need check for refreshed settings - if (!event.indexMetaDataChanged(indexMetaData)) { - continue; - } - Index index = indexMetaData.getIndex(); - IndexService indexService = indicesService.indexService(index); - if (indexService == null) { - // already deleted on us, ignore it - continue; - } - indexService.updateMetaData(indexMetaData); - } - } - - private void applyNewIndices(final ClusterChangedEvent event) { + private void createIndices(final ClusterState state) { // we only create indices for shards that are allocated - RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId()); - if (routingNode == null) { + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + if (localRoutingNode == null) { return; } - for (ShardRouting shard : routingNode) { - if (!indicesService.hasIndex(shard.index())) { - final IndexMetaData indexMetaData = event.state().metaData().getIndexSafe(shard.index()); - if (logger.isDebugEnabled()) { - logger.debug("[{}] creating index", indexMetaData.getIndex()); - } - try { - indicesService.createIndex(nodeServicesProvider, indexMetaData, buildInIndexListener); - } catch (Throwable e) { - sendFailShard(shard, "failed to create index", e); + // create map of indices to create with shards to fail if index creation fails + final Map> indicesToCreate = new HashMap<>(); + for (ShardRouting shardRouting : localRoutingNode) { + if (failedShardsCache.containsKey(shardRouting.shardId()) == false) { + final Index index = shardRouting.index(); + if (indicesService.indexService(index) == null) { + indicesToCreate.computeIfAbsent(index, k -> new ArrayList<>()).add(shardRouting); } } } - } - private void applyMappings(ClusterChangedEvent event) { - // go over and update mappings - for (IndexMetaData indexMetaData : event.state().metaData()) { - Index index = indexMetaData.getIndex(); - if (!indicesService.hasIndex(index)) { - // we only create / update here - continue; - } - boolean requireRefresh = false; - IndexService indexService = indicesService.indexService(index); - if (indexService == null) { - // got deleted on us, ignore (closing the node) - return; - } + for (Map.Entry> entry : indicesToCreate.entrySet()) { + final Index index = entry.getKey(); + final IndexMetaData indexMetaData = state.metaData().index(index); + logger.debug("[{}] creating index", index); + + AllocatedIndex indexService = null; try { - MapperService mapperService = indexService.mapperService(); - // go over and add the relevant mappings (or update them) - for (ObjectCursor cursor : indexMetaData.getMappings().values()) { - MappingMetaData mappingMd = cursor.value; - String mappingType = mappingMd.type(); - CompressedXContent mappingSource = mappingMd.source(); - requireRefresh |= processMapping(index.getName(), mapperService, mappingType, mappingSource); - } - if (requireRefresh && sendRefreshMapping) { - nodeMappingRefreshAction.nodeMappingRefresh(event.state(), - new NodeMappingRefreshAction.NodeMappingRefreshRequest(index.getName(), indexMetaData.getIndexUUID(), - event.state().nodes().getLocalNodeId()) + indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, buildInIndexListener); + if (indexService.updateMapping(indexMetaData) && sendRefreshMapping) { + nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), + new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(), + indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId()) ); } } catch (Throwable t) { - // if we failed the mappings anywhere, we need to fail the shards for this index, note, we safeguard - // by creating the processing the mappings on the master, or on the node the mapping was introduced on, - // so this failure typically means wrong node level configuration or something similar - for (IndexShard indexShard : indexService) { - ShardRouting shardRouting = indexShard.routingEntry(); - failAndRemoveShard(shardRouting, indexService, true, "failed to update mappings", t); - } - } - } - } - - private boolean processMapping(String index, MapperService mapperService, String mappingType, CompressedXContent mappingSource) throws Throwable { - // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same - // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the - // merge version of it, which it does when refreshing the mappings), and warn log it. - boolean requiresRefresh = false; - try { - DocumentMapper existingMapper = mapperService.documentMapper(mappingType); - - if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) { - String op = existingMapper == null ? "adding" : "updating"; - if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) { - logger.debug("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); - } else if (logger.isTraceEnabled()) { - logger.trace("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string()); + final String failShardReason; + if (indexService == null) { + failShardReason = "failed to create index"; } else { - logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index, op, mappingType); + failShardReason = "failed to update mapping for index"; + indicesService.removeIndex(index, "removing index (mapping update failed)"); } - mapperService.merge(mappingType, mappingSource, MapperService.MergeReason.MAPPING_RECOVERY, true); - if (!mapperService.documentMapper(mappingType).mappingSource().equals(mappingSource)) { - logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index, mappingType, mappingSource, mapperService.documentMapper(mappingType).mappingSource()); - requiresRefresh = true; + for (ShardRouting shardRouting : entry.getValue()) { + sendFailShard(shardRouting, failShardReason, t); } } - } catch (Throwable e) { - logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index, mappingType, mappingSource); - throw e; } - return requiresRefresh; } - - private void applyNewOrUpdatedShards(final ClusterChangedEvent event) { - if (!indicesService.changesAllowed()) { + private void updateIndices(ClusterChangedEvent event) { + if (!event.metaDataChanged()) { return; } - - RoutingTable routingTable = event.state().routingTable(); - RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId()); - - if (routingNode == null) { - failedShards.clear(); - return; - } - - DiscoveryNodes nodes = event.state().nodes(); - for (final ShardRouting shardRouting : routingNode) { - final IndexService indexService = indicesService.indexService(shardRouting.index()); - if (indexService == null) { - // creation failed for some reasons - assert failedShards.containsKey(shardRouting.shardId()) : - "index has local allocation but is not created by applyNewIndices and is not failed " + shardRouting; - continue; - } - final IndexMetaData indexMetaData = event.state().metaData().index(shardRouting.index()); - assert indexMetaData != null : "index has local allocation but no meta data. " + shardRouting.index(); - - final int shardId = shardRouting.id(); - - if (!indexService.hasShard(shardId) && shardRouting.started()) { - if (failedShards.containsKey(shardRouting.shardId())) { - if (nodes.getMasterNode() != null) { - String message = "master " + nodes.getMasterNode() + " marked shard as started, but shard has previous failed. resending shard failure"; - logger.trace("[{}] re-sending failed shard [{}], reason [{}]", shardRouting.shardId(), shardRouting, message); - shardStateAction.shardFailed(shardRouting, shardRouting, message, null, SHARD_STATE_ACTION_LISTENER); + final ClusterState state = event.state(); + for (AllocatedIndex indexService : indicesService) { + final Index index = indexService.index(); + final IndexMetaData currentIndexMetaData = indexService.getIndexSettings().getIndexMetaData(); + final IndexMetaData newIndexMetaData = state.metaData().index(index); + assert newIndexMetaData != null : "index " + index + " should have been removed by deleteIndices"; + if (ClusterChangedEvent.indexMetaDataChanged(currentIndexMetaData, newIndexMetaData)) { + indexService.updateMetaData(newIndexMetaData); + try { + if (indexService.updateMapping(newIndexMetaData) && sendRefreshMapping) { + nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), + new NodeMappingRefreshAction.NodeMappingRefreshRequest(newIndexMetaData.getIndex().getName(), + newIndexMetaData.getIndexUUID(), state.nodes().getLocalNodeId()) + ); } - } else { - // the master thinks we are started, but we don't have this shard at all, mark it as failed - sendFailShard(shardRouting, "master [" + nodes.getMasterNode() + "] marked shard as started, but shard has not been created, mark shard as failed", null); - } - continue; - } + } catch (Throwable t) { + indicesService.removeIndex(indexService.index(), "removing index (mapping update failed)"); - IndexShard indexShard = indexService.getShardOrNull(shardId); - if (indexShard != null) { - ShardRouting currentRoutingEntry = indexShard.routingEntry(); - // if the current and global routing are initializing, but are still not the same, its a different "shard" being allocated - // for example: a shard that recovers from one node and now needs to recover to another node, - // or a replica allocated and then allocating a primary because the primary failed on another node - boolean shardHasBeenRemoved = false; - assert currentRoutingEntry.isSameAllocation(shardRouting) : - "local shard has a different allocation id but wasn't cleaning by applyDeletedShards. " - + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; - if (shardRouting.isPeerRecovery()) { - RecoveryState recoveryState = indexShard.recoveryState(); - final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting); - if (recoveryState.getSourceNode().equals(sourceNode) == false) { - if (recoveryTargetService.cancelRecoveriesForShard(currentRoutingEntry.shardId(), "recovery source node changed")) { - // getting here means that the shard was still recovering - logger.debug("[{}][{}] removing shard (recovery source changed), current [{}], global [{}])", shardRouting.index(), shardRouting.id(), currentRoutingEntry, shardRouting); - indexService.removeShard(shardRouting.id(), "removing shard (recovery source node changed)"); - shardHasBeenRemoved = true; + // fail shards that would be created or updated by createOrUpdateShards + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + if (localRoutingNode != null) { + for (final ShardRouting shardRouting : localRoutingNode) { + if (shardRouting.index().equals(index) && failedShardsCache.containsKey(shardRouting.shardId()) == false) { + sendFailShard(shardRouting, "failed to update mapping for index", t); + } } } } - - if (shardHasBeenRemoved == false) { - try { - indexShard.updateRoutingEntry(shardRouting, event.state().blocks().disableStatePersistence() == false); - } catch (Throwable e) { - failAndRemoveShard(shardRouting, indexService, true, "failed updating shard routing entry", e); - } - } - } - - if (shardRouting.initializing()) { - applyInitializingShard(event.state(), indexService, shardRouting); } } } - private void applyInitializingShard(final ClusterState state, IndexService indexService, final ShardRouting shardRouting) { - final RoutingTable routingTable = state.routingTable(); - final DiscoveryNodes nodes = state.getNodes(); - final int shardId = shardRouting.id(); + private void createOrUpdateShards(final ClusterState state) { + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); + if (localRoutingNode == null) { + return; + } - if (indexService.hasShard(shardId)) { - IndexShard indexShard = indexService.getShard(shardId); - if (indexShard.state() == IndexShardState.STARTED || indexShard.state() == IndexShardState.POST_RECOVERY) { - // the master thinks we are initializing, but we are already started or on POST_RECOVERY and waiting - // for master to confirm a shard started message (either master failover, or a cluster event before - // we managed to tell the master we started), mark us as started - if (logger.isTraceEnabled()) { - logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}", - indexShard.shardId(), indexShard.state(), nodes.getMasterNode()); - } - if (nodes.getMasterNode() != null) { - shardStateAction.shardStarted(shardRouting, - "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + indexShard.state() + "], mark shard as started", - SHARD_STATE_ACTION_LISTENER); - } - return; - } else { - if (indexShard.ignoreRecoveryAttempt()) { - logger.trace("ignoring recovery instruction for an existing shard {} (shard state: [{}])", indexShard.shardId(), indexShard.state()); - return; + DiscoveryNodes nodes = state.nodes(); + RoutingTable routingTable = state.routingTable(); + + for (final ShardRouting shardRouting : localRoutingNode) { + ShardId shardId = shardRouting.shardId(); + if (failedShardsCache.containsKey(shardId) == false) { + AllocatedIndex indexService = indicesService.indexService(shardId.getIndex()); + assert indexService != null : "index " + shardId.getIndex() + " should have been created by createIndices"; + Shard shard = indexService.getShardOrNull(shardId.id()); + if (shard == null) { + assert shardRouting.initializing() : shardRouting + " should have been removed by failMissingShards"; + createShard(nodes, routingTable, shardRouting, indexService); + } else { + updateShard(nodes, shardRouting, shard); } } } + } + + private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting, + AllocatedIndex indexService) { + assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting; - // if we're in peer recovery, try to find out the source node now so in case it fails, we will not create the index shard DiscoveryNode sourceNode = null; if (shardRouting.isPeerRecovery()) { sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting); @@ -595,50 +516,49 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent { - try { - nodeServicesProvider.getClient().admin().indices().preparePutMapping() - .setConcreteIndex(indexService.index()) // concrete index - no name clash, it uses uuid - .setType(type) - .setSource(mapping.source().string()) - .get(); - } catch (IOException ex) { - throw new ElasticsearchException("failed to stringify mapping source", ex); - } - }, indicesService); + final IndexShardState state = shard.state(); + if (shardRouting.initializing() && (state == IndexShardState.STARTED || state == IndexShardState.POST_RECOVERY)) { + // the master thinks we are initializing, but we are already started or on POST_RECOVERY and waiting + // for master to confirm a shard started message (either master failover, or a cluster event before + // we managed to tell the master we started), mark us as started + if (logger.isTraceEnabled()) { + logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}", + shardRouting.shardId(), state, nodes.getMasterNode()); + } + if (nodes.getMasterNode() != null) { + shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() + + " marked shard as initializing, but shard state is [" + state + "], mark shard as started", + SHARD_STATE_ACTION_LISTENER); + } + } } /** @@ -646,7 +566,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent indexService = indicesService.indexService(shardRouting.shardId().getIndex()); + if (indexService != null) { + indexService.removeShard(shardRouting.shardId().id(), message); } + } catch (ShardNotFoundException e) { + // the node got closed on us, ignore it + } catch (Throwable e1) { + logger.warn("[{}][{}] failed to remove shard after failure ([{}])", e1, shardRouting.getIndexName(), shardRouting.getId(), + message); } if (sendShardFailure) { sendFailShard(shardRouting, message, failure); @@ -760,23 +683,156 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent { @Override public void handle(final IndexShard.ShardFailure shardFailure) { - final IndexService indexService = indicesService.indexService(shardFailure.routing.shardId().getIndex()); final ShardRouting shardRouting = shardFailure.routing; threadPool.generic().execute(() -> { - synchronized (mutex) { - failAndRemoveShard(shardRouting, indexService, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause); + synchronized (IndicesClusterStateService.this) { + failAndRemoveShard(shardRouting, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause); } }); } } + + public interface Shard { + + /** + * Returns the shard id of this shard. + */ + ShardId shardId(); + + /** + * Returns the latest cluster routing entry received with this shard. + */ + ShardRouting routingEntry(); + + /** + * Returns the latest internal shard state. + */ + IndexShardState state(); + + /** + * Returns the recovery state associated with this shard. + */ + RecoveryState recoveryState(); + + /** + * Updates the shards routing entry. This mutate the shards internal state depending + * on the changes that get introduced by the new routing value. This method will persist shard level metadata. + * + * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted + * @throws IOException if shard state could not be persisted + */ + void updateRoutingEntry(ShardRouting shardRouting) throws IOException; + } + + public interface AllocatedIndex extends Iterable, IndexComponent { + + /** + * Returns the index settings of this index. + */ + IndexSettings getIndexSettings(); + + /** + * Updates the meta data of this index. Changes become visible through {@link #getIndexSettings()} + */ + void updateMetaData(IndexMetaData indexMetaData); + + /** + * Checks if index requires refresh from master. + */ + boolean updateMapping(IndexMetaData indexMetaData) throws IOException; + + /** + * Returns shard with given id. + */ + @Nullable T getShardOrNull(int shardId); + + /** + * Removes shard with given id. + */ + void removeShard(int shardId, String message); + } + + public interface AllocatedIndices> extends Iterable { + + /** + * Creates a new {@link IndexService} for the given metadata. + * @param indexMetaData the index metadata to create the index for + * @param builtInIndexListener a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with + * the per-index listeners + * @throws IndexAlreadyExistsException if the index already exists. + */ + U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, + List builtInIndexListener) throws IOException; + + /** + * Verify that the contents on disk for the given index is deleted; if not, delete the contents. + * This method assumes that an index is already deleted in the cluster state and/or explicitly + * through index tombstones. + * @param index {@code Index} to make sure its deleted from disk + * @param clusterState {@code ClusterState} to ensure the index is not part of it + * @return IndexMetaData for the index loaded from disk + */ + IndexMetaData verifyIndexIsDeleted(Index index, ClusterState clusterState); + + /** + * Deletes the given index. Persistent parts of the index + * like the shards files, state and transaction logs are removed once all resources are released. + * + * Equivalent to {@link #removeIndex(Index, String)} but fires + * different lifecycle events to ensure pending resources of this index are immediately removed. + * @param index the index to delete + * @param reason the high level reason causing this delete + */ + void deleteIndex(Index index, String reason); + + /** + * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index + * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)} + */ + void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState); + + /** + * Removes the given index from this service and releases all associated resources. Persistent parts of the index + * like the shards files, state and transaction logs are kept around in the case of a disaster recovery. + * @param index the index to remove + * @param reason the high level reason causing this removal + */ + void removeIndex(Index index, String reason); + + /** + * Returns an IndexService for the specified index if exists otherwise returns null. + */ + @Nullable U indexService(Index index); + + /** + * Creates shard for the specified shard routing and starts recovery, + */ + T createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService, + RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, + NodeServicesProvider nodeServicesProvider, Callback onShardFailure) throws IOException; + + /** + * Returns shard for the specified id if it exists otherwise returns null. + */ + default T getShardOrNull(ShardId shardId) { + U indexRef = indexService(shardId.getIndex()); + if (indexRef != null) { + return indexRef.getShardOrNull(shardId.id()); + } + return null; + } + + void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeValue) throws IOException, InterruptedException; + } } diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java index f3cb76199dc..16e4d62a721 100644 --- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java +++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java @@ -114,11 +114,9 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL final ClusterState state = clusterService.state(); final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, indicesOptions, aliasesOrIndices); final Map> results = ConcurrentCollections.newConcurrentMap(); - int totalNumberOfShards = 0; int numberOfShards = 0; for (Index index : concreteIndices) { final IndexMetaData indexMetaData = state.metaData().getIndexSafe(index); - totalNumberOfShards += indexMetaData.getTotalNumberOfShards(); numberOfShards += indexMetaData.getNumberOfShards(); results.put(index.getName(), Collections.synchronizedList(new ArrayList<>())); @@ -127,7 +125,6 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL listener.onResponse(new SyncedFlushResponse(results)); return; } - final int finalTotalNumberOfShards = totalNumberOfShards; final CountDown countDown = new CountDown(numberOfShards); for (final Index concreteIndex : concreteIndices) { @@ -136,7 +133,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL final int indexNumberOfShards = indexMetaData.getNumberOfShards(); for (int shard = 0; shard < indexNumberOfShards; shard++) { final ShardId shardId = new ShardId(indexMetaData.getIndex(), shard); - attemptSyncedFlush(shardId, new ActionListener() { + innerAttemptSyncedFlush(shardId, state, new ActionListener() { @Override public void onResponse(ShardsSyncedFlushResult syncedFlushResult) { results.get(index).add(syncedFlushResult); @@ -148,7 +145,8 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL @Override public void onFailure(Throwable e) { logger.debug("{} unexpected error while executing synced flush", shardId); - results.get(index).add(new ShardsSyncedFlushResult(shardId, finalTotalNumberOfShards, e.getMessage())); + final int totalShards = indexMetaData.getNumberOfReplicas() + 1; + results.get(index).add(new ShardsSyncedFlushResult(shardId, totalShards, e.getMessage())); if (countDown.countDown()) { listener.onResponse(new SyncedFlushResponse(results)); } @@ -185,8 +183,11 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL * Synced flush is a best effort operation. The sync id may be written on all, some or none of the copies. **/ public void attemptSyncedFlush(final ShardId shardId, final ActionListener actionListener) { + innerAttemptSyncedFlush(shardId, clusterService.state(), actionListener); + } + + private void innerAttemptSyncedFlush(final ShardId shardId, final ClusterState state, final ActionListener actionListener) { try { - final ClusterState state = clusterService.state(); final IndexShardRoutingTable shardRoutingTable = getShardRoutingTable(shardId, state); final List activeShards = shardRoutingTable.activeShards(); final int totalShards = shardRoutingTable.getSize(); diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java index 8fd08d9f8fb..49cdb737ed3 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java @@ -20,7 +20,6 @@ package org.elasticsearch.indices.recovery; import org.apache.lucene.util.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -76,7 +75,6 @@ public final class RecoveryFileChunkRequest extends TransportRequest { return position; } - @Nullable public String checksum() { return metaData.checksum(); } @@ -105,11 +103,10 @@ public final class RecoveryFileChunkRequest extends TransportRequest { String name = in.readString(); position = in.readVLong(); long length = in.readVLong(); - String checksum = in.readOptionalString(); + String checksum = in.readString(); content = in.readBytesReference(); - Version writtenBy = null; - String versionString = in.readOptionalString(); - writtenBy = Lucene.parseVersionLenient(versionString, null); + Version writtenBy = Lucene.parseVersionLenient(in.readString(), null); + assert writtenBy != null; metaData = new StoreFileMetaData(name, length, checksum, writtenBy); lastChunk = in.readBoolean(); totalTranslogOps = in.readVInt(); @@ -124,9 +121,9 @@ public final class RecoveryFileChunkRequest extends TransportRequest { out.writeString(metaData.name()); out.writeVLong(position); out.writeVLong(metaData.length()); - out.writeOptionalString(metaData.checksum()); + out.writeString(metaData.checksum()); out.writeBytesReference(content); - out.writeOptionalString(metaData.writtenBy() == null ? null : metaData.writtenBy().toString()); + out.writeString(metaData.writtenBy().toString()); out.writeBoolean(lastChunk); out.writeVInt(totalTranslogOps); out.writeLong(sourceThrottleTimeInNanos); diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 04063ce5864..b6bfba04ff5 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -97,7 +97,7 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchService; import org.elasticsearch.snapshots.SnapshotShardsService; import org.elasticsearch.snapshots.SnapshotsService; -import org.elasticsearch.tasks.TaskResultsService; +import org.elasticsearch.tasks.TaskPersistenceService; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolModule; @@ -340,7 +340,7 @@ public class Node implements Closeable { // Start the transport service now so the publish address will be added to the local disco node in ClusterService TransportService transportService = injector.getInstance(TransportService.class); - transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class)); + transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskPersistenceService.class)); transportService.start(); validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress()); diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 9e17eec2e93..643998f19c2 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -237,6 +237,10 @@ class InstallPluginCommand extends SettingCommand { } // fall back to plain old URL + if (pluginId.contains(":/") == false) { + // definitely not a valid url, so assume it is a plugin name + throw new UserError(ExitCodes.USAGE, "Unknown plugin " + pluginId); + } terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8")); return downloadZip(terminal, pluginId, tmpDir); } diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java index 23d390dfcfe..04900705e0a 100644 --- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java +++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java @@ -115,4 +115,5 @@ public abstract class BlobStoreFormat { } } + } diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java index aa567af09f7..be72ed19c40 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestController.java +++ b/core/src/main/java/org/elasticsearch/rest/RestController.java @@ -113,30 +113,14 @@ public class RestController extends AbstractLifecycleComponent { } /** - * Registers a rest handler to be execute when the provided method and path match the request. + * Registers a rest handler to be executed when the provided method and path match the request. */ public void registerHandler(RestRequest.Method method, String path, RestHandler handler) { - switch (method) { - case GET: - getHandlers.insert(path, handler); - break; - case DELETE: - deleteHandlers.insert(path, handler); - break; - case POST: - postHandlers.insert(path, handler); - break; - case PUT: - putHandlers.insert(path, handler); - break; - case OPTIONS: - optionsHandlers.insert(path, handler); - break; - case HEAD: - headHandlers.insert(path, handler); - break; - default: - throw new IllegalArgumentException("Can't handle [" + method + "] for path [" + path + "]"); + PathTrie handlers = getHandlersForMethod(method); + if (handlers != null) { + handlers.insert(path, handler); + } else { + throw new IllegalArgumentException("Can't handle [" + method + "] for path [" + path + "]"); } } @@ -159,6 +143,15 @@ public class RestController extends AbstractLifecycleComponent { return new ControllerFilterChain(executionFilter); } + /** + * @param request The current request. Must not be null. + * @return true iff the circuit breaker limit must be enforced for processing this request. + */ + public boolean canTripCircuitBreaker(RestRequest request) { + RestHandler handler = getHandler(request); + return (handler != null) ? handler.canTripCircuitBreaker() : true; + } + public void dispatchRequest(final RestRequest request, final RestChannel channel, ThreadContext threadContext) throws Exception { if (!checkRequestParameters(request, channel)) { return; @@ -226,19 +219,27 @@ public class RestController extends AbstractLifecycleComponent { private RestHandler getHandler(RestRequest request) { String path = getPath(request); - RestRequest.Method method = request.method(); + PathTrie handlers = getHandlersForMethod(request.method()); + if (handlers != null) { + return handlers.retrieve(path, request.params()); + } else { + return null; + } + } + + private PathTrie getHandlersForMethod(RestRequest.Method method) { if (method == RestRequest.Method.GET) { - return getHandlers.retrieve(path, request.params()); + return getHandlers; } else if (method == RestRequest.Method.POST) { - return postHandlers.retrieve(path, request.params()); + return postHandlers; } else if (method == RestRequest.Method.PUT) { - return putHandlers.retrieve(path, request.params()); + return putHandlers; } else if (method == RestRequest.Method.DELETE) { - return deleteHandlers.retrieve(path, request.params()); + return deleteHandlers; } else if (method == RestRequest.Method.HEAD) { - return headHandlers.retrieve(path, request.params()); + return headHandlers; } else if (method == RestRequest.Method.OPTIONS) { - return optionsHandlers.retrieve(path, request.params()); + return optionsHandlers; } else { return null; } diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java index 45a172e816d..31970441493 100644 --- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -25,4 +25,8 @@ package org.elasticsearch.rest; public interface RestHandler { void handleRequest(RestRequest request, RestChannel channel) throws Exception; -} \ No newline at end of file + + default boolean canTripCircuitBreaker() { + return true; + } +} diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java index ccd0f982597..a2c6ffeaf14 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java @@ -64,4 +64,9 @@ public class RestClusterHealthAction extends BaseRestHandler { clusterHealthRequest.waitForNodes(request.param("wait_for_nodes", clusterHealthRequest.waitForNodes())); client.admin().cluster().health(clusterHealthRequest, new RestStatusToXContentListener(channel)); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java index 19bee19509a..2030d5e7d92 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java @@ -78,4 +78,9 @@ public class RestNodesHotThreadsAction extends BaseRestHandler { } }); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java index 65b7715385f..0f1e8691770 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java @@ -103,4 +103,9 @@ public class RestNodesInfoAction extends BaseRestHandler { client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel)); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java index 47cce5283a9..8019de0b4e5 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java @@ -115,4 +115,9 @@ public class RestNodesStatsAction extends BaseRestHandler { client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel)); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java index 658090bb6db..0602abe651f 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java @@ -19,9 +19,11 @@ package org.elasticsearch.rest.action.admin.cluster.node.tasks; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -33,13 +35,16 @@ import org.elasticsearch.rest.action.support.RestToXContentListener; import org.elasticsearch.tasks.TaskId; import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.nodeSettingListener; public class RestCancelTasksAction extends BaseRestHandler { + private final ClusterService clusterService; @Inject - public RestCancelTasksAction(Settings settings, RestController controller, Client client) { + public RestCancelTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) { super(settings, client); + this.clusterService = clusterService; controller.registerHandler(POST, "/_tasks/_cancel", this); controller.registerHandler(POST, "/_tasks/{taskId}/_cancel", this); } @@ -56,6 +61,12 @@ public class RestCancelTasksAction extends BaseRestHandler { cancelTasksRequest.setNodesIds(nodesIds); cancelTasksRequest.setActions(actions); cancelTasksRequest.setParentTaskId(parentTaskId); - client.admin().cluster().cancelTasks(cancelTasksRequest, new RestToXContentListener<>(channel)); + ActionListener listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel)); + client.admin().cluster().cancelTasks(cancelTasksRequest, listener); + } + + @Override + public boolean canTripCircuitBreaker() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java new file mode 100644 index 00000000000..e5617711014 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.cluster.node.tasks; + +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.support.RestToXContentListener; +import org.elasticsearch.tasks.TaskId; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestGetTaskAction extends BaseRestHandler { + @Inject + public RestGetTaskAction(Settings settings, RestController controller, Client client) { + super(settings, client); + controller.registerHandler(GET, "/_tasks/{taskId}", this); + } + + @Override + public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { + TaskId taskId = new TaskId(request.param("taskId")); + boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false); + TimeValue timeout = request.paramAsTime("timeout", null); + + GetTaskRequest getTaskRequest = new GetTaskRequest(); + getTaskRequest.setTaskId(taskId); + getTaskRequest.setWaitForCompletion(waitForCompletion); + getTaskRequest.setTimeout(timeout); + client.admin().cluster().getTask(getTaskRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java index 1227e7c10d5..8fa13e808ac 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java @@ -19,8 +19,11 @@ package org.elasticsearch.rest.action.admin.cluster.node.tasks; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -36,25 +39,24 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestListTasksAction extends BaseRestHandler { + private final ClusterService clusterService; @Inject - public RestListTasksAction(Settings settings, RestController controller, Client client) { + public RestListTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) { super(settings, client); + this.clusterService = clusterService; controller.registerHandler(GET, "/_tasks", this); - controller.registerHandler(GET, "/_tasks/{taskId}", this); } public static ListTasksRequest generateListTasksRequest(RestRequest request) { boolean detailed = request.paramAsBoolean("detailed", false); String[] nodesIds = Strings.splitStringByCommaToArray(request.param("node_id")); - TaskId taskId = new TaskId(request.param("taskId", request.param("task_id"))); String[] actions = Strings.splitStringByCommaToArray(request.param("actions")); TaskId parentTaskId = new TaskId(request.param("parent_task_id")); boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false); TimeValue timeout = request.paramAsTime("timeout", null); ListTasksRequest listTasksRequest = new ListTasksRequest(); - listTasksRequest.setTaskId(taskId); listTasksRequest.setNodesIds(nodesIds); listTasksRequest.setDetailed(detailed); listTasksRequest.setActions(actions); @@ -66,6 +68,32 @@ public class RestListTasksAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { - client.admin().cluster().listTasks(generateListTasksRequest(request), new RestToXContentListener<>(channel)); + ActionListener listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel)); + client.admin().cluster().listTasks(generateListTasksRequest(request), listener); + } + + /** + * Wrap the normal channel listener in one that sets the discovery nodes on the response so we can support all of it's toXContent + * formats. + */ + public static ActionListener nodeSettingListener(ClusterService clusterService, + ActionListener channelListener) { + return new ActionListener() { + @Override + public void onResponse(T response) { + response.setDiscoveryNodes(clusterService.state().nodes()); + channelListener.onResponse(response); + } + + @Override + public void onFailure(Throwable e) { + channelListener.onFailure(e); + } + }; + } + + @Override + public boolean canTripCircuitBreaker() { + return false; } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java index 62779292ac1..44a7f2f714b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java @@ -72,6 +72,11 @@ public class RestClusterGetSettingsAction extends BaseRestHandler { }); } + @Override + public boolean canTripCircuitBreaker() { + return false; + } + private XContentBuilder renderResponse(ClusterState state, boolean renderDefaults, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java index 64083f1e806..b25866d4520 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java @@ -76,4 +76,9 @@ public class RestClusterUpdateSettingsAction extends BaseRestHandler { } }); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java index edce416072e..c756796446e 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java @@ -96,6 +96,11 @@ public class RestClusterStateAction extends BaseRestHandler { }); } + @Override + public boolean canTripCircuitBreaker() { + return false; + } + static final class Fields { static final String CLUSTER_NAME = "cluster_name"; } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java index 29cc6377494..c83a5b7ae1b 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java @@ -47,4 +47,9 @@ public class RestClusterStatsAction extends BaseRestHandler { clusterStatsRequest.timeout(request.param("timeout")); client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java index 7adb6909532..97e424445d8 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java @@ -75,6 +75,11 @@ public class RestClearIndicesCacheAction extends BaseRestHandler { }); } + @Override + public boolean canTripCircuitBreaker() { + return false; + } + public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest, ParseFieldMatcher parseFieldMatcher) { for (Map.Entry entry : request.params().entrySet()) { diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java index 55ed1d8dda4..623e60cb4b9 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java @@ -117,4 +117,9 @@ public class RestIndicesStatsAction extends BaseRestHandler { } }); } + + @Override + public boolean canTripCircuitBreaker() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java index 9e150e6178a..9bb01a7b166 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java @@ -19,12 +19,12 @@ package org.elasticsearch.rest.action.cat; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; import org.elasticsearch.common.inject.Inject; @@ -37,7 +37,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.support.RestResponseListener; import org.elasticsearch.rest.action.support.RestTable; -import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -48,11 +48,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.generateListTasksRequest; public class RestTasksAction extends AbstractCatAction { + private final ClusterService clusterService; @Inject - public RestTasksAction(Settings settings, RestController controller, Client client) { + public RestTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) { super(settings, controller, client); controller.registerHandler(GET, "/_cat/tasks", this); + this.clusterService = clusterService; } @Override @@ -104,9 +106,10 @@ public class RestTasksAction extends AbstractCatAction { private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss"); - private void buildRow(Table table, boolean fullId, boolean detailed, TaskInfo taskInfo) { + private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNodes discoveryNodes, TaskInfo taskInfo) { table.startRow(); - DiscoveryNode node = taskInfo.getNode(); + String nodeId = taskInfo.getTaskId().getNodeId(); + DiscoveryNode node = discoveryNodes.get(nodeId); table.addCell(taskInfo.getId()); table.addCell(taskInfo.getAction()); @@ -122,15 +125,16 @@ public class RestTasksAction extends AbstractCatAction { table.addCell(taskInfo.getRunningTimeNanos()); table.addCell(TimeValue.timeValueNanos(taskInfo.getRunningTimeNanos()).toString()); - table.addCell(fullId ? node.getId() : Strings.substring(node.getId(), 0, 4)); - table.addCell(node.getHostAddress()); - if (node.getAddress() instanceof InetSocketTransportAddress) { + // Node information. Note that the node may be null because it has left the cluster between when we got this response and now. + table.addCell(fullId ? nodeId : Strings.substring(nodeId, 0, 4)); + table.addCell(node == null ? "-" : node.getHostAddress()); + if (node != null && node.getAddress() instanceof InetSocketTransportAddress) { table.addCell(((InetSocketTransportAddress) node.getAddress()).address().getPort()); } else { table.addCell("-"); } - table.addCell(node.getName()); - table.addCell(node.getVersion().toString()); + table.addCell(node == null ? "-" : node.getName()); + table.addCell(node == null ? "-" : node.getVersion().toString()); if (detailed) { table.addCell(taskInfo.getDescription()); @@ -139,10 +143,11 @@ public class RestTasksAction extends AbstractCatAction { } private void buildGroups(Table table, boolean detailed, boolean fullId, List taskGroups) { + DiscoveryNodes discoveryNodes = clusterService.state().nodes(); List sortedGroups = new ArrayList<>(taskGroups); sortedGroups.sort((o1, o2) -> Long.compare(o1.getTaskInfo().getStartTime(), o2.getTaskInfo().getStartTime())); for (TaskGroup taskGroup : sortedGroups) { - buildRow(table, fullId, detailed, taskGroup.getTaskInfo()); + buildRow(table, fullId, detailed, discoveryNodes, taskGroup.getTaskInfo()); buildGroups(table, fullId, detailed, taskGroup.getChildTasks()); } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 8c76525c857..2050360a9d5 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -91,6 +91,9 @@ public class RestMultiSearchAction extends BaseRestHandler { @Override public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + if (request.hasParam("max_concurrent_searches")) { + multiSearchRequest.maxConcurrentSearchRequests(request.paramAsInt("max_concurrent_searches", 0)); + } String[] indices = Strings.splitStringByCommaToArray(request.param("index")); String[] types = Strings.splitStringByCommaToArray(request.param("type")); diff --git a/core/src/main/java/org/elasticsearch/search/Highlighters.java b/core/src/main/java/org/elasticsearch/search/Highlighters.java new file mode 100644 index 00000000000..b66a6bda640 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/Highlighters.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.highlight.FastVectorHighlighter; +import org.elasticsearch.search.highlight.Highlighter; +import org.elasticsearch.search.highlight.PlainHighlighter; +import org.elasticsearch.search.highlight.PostingsHighlighter; + +import java.util.HashMap; +import java.util.Map; + +/** + * An extensions point and registry for all the highlighters a node supports. + */ +public final class Highlighters { + + private final Map parsers = new HashMap<>(); + + public Highlighters(Settings settings) { + registerHighlighter("fvh", new FastVectorHighlighter(settings)); + registerHighlighter("plain", new PlainHighlighter()); + registerHighlighter("postings", new PostingsHighlighter()); + } + + /** + * Returns the highlighter for the given key or null if there is no highlighter registered for that key. + */ + public Highlighter get(String key) { + return parsers.get(key); + } + + /** + * Registers a highlighter for the given key + * @param key the key the highlighter should be referenced by in the search request + * @param highlighter the highlighter instance + */ + void registerHighlighter(String key, Highlighter highlighter) { + if (highlighter == null) { + throw new IllegalArgumentException("Can't register null highlighter for key: [" + key + "]"); + } + if (parsers.putIfAbsent(key, highlighter) != null) { + throw new IllegalArgumentException("Can't register the same [highlighter] more than once for [" + key + "]"); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/Scroll.java b/core/src/main/java/org/elasticsearch/search/Scroll.java index 2d703bdae90..13298b2cbbe 100644 --- a/core/src/main/java/org/elasticsearch/search/Scroll.java +++ b/core/src/main/java/org/elasticsearch/search/Scroll.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; -import static org.elasticsearch.common.unit.TimeValue.readTimeValue; - /** * A scroll enables scrolling of search request. It holds a {@link #keepAlive()} time that * will control how long to keep the scrolling resources open. @@ -64,18 +62,11 @@ public class Scroll implements Streamable { @Override public void readFrom(StreamInput in) throws IOException { - if (in.readBoolean()) { - keepAlive = readTimeValue(in); - } + in.readOptionalWriteable(TimeValue::new); } @Override public void writeTo(StreamOutput out) throws IOException { - if (keepAlive == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - keepAlive.writeTo(out); - } + out.writeOptionalWriteable(keepAlive); } } diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java index 8afda85c58f..2b212756528 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java @@ -24,11 +24,9 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ParseFieldRegistry; @@ -93,7 +91,6 @@ import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.WeightBuilder; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.action.SearchTransportService; -import org.elasticsearch.search.aggregations.AggregationPhase; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorParsers; @@ -237,12 +234,10 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel; import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregatorBuilder; import org.elasticsearch.search.controller.SearchPhaseController; -import org.elasticsearch.search.dfs.DfsPhase; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase; import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase; -import org.elasticsearch.search.fetch.innerhits.InnerHitsFetchSubPhase; import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase; import org.elasticsearch.search.fetch.parent.ParentFieldSubFetchPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; @@ -250,8 +245,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceSubPhase; import org.elasticsearch.search.fetch.version.VersionFetchSubPhase; import org.elasticsearch.search.highlight.HighlightPhase; import org.elasticsearch.search.highlight.Highlighter; -import org.elasticsearch.search.highlight.Highlighters; -import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -263,6 +256,7 @@ import org.elasticsearch.search.suggest.Suggester; import org.elasticsearch.search.suggest.Suggesters; import java.util.HashSet; +import java.util.Objects; import java.util.Set; /** @@ -270,9 +264,8 @@ import java.util.Set; */ public class SearchModule extends AbstractModule { - private final Highlighters highlighters = new Highlighters(); + private final Highlighters highlighters; private final Suggesters suggesters; - private final ParseFieldRegistry> scoreFunctionParserRegistry = new ParseFieldRegistry<>("score_function"); private final IndicesQueriesRegistry queryParserRegistry = new IndicesQueriesRegistry(); private final ParseFieldRegistry aggregationParserRegistry = new ParseFieldRegistry<>("aggregation"); @@ -284,7 +277,7 @@ public class SearchModule extends AbstractModule { private final ParseFieldRegistry movingAverageModelParserRegistry = new ParseFieldRegistry<>( "moving_avg_model"); - private final Set> fetchSubPhases = new HashSet<>(); + private final Set fetchSubPhases = new HashSet<>(); private final Settings settings; private final NamedWriteableRegistry namedWriteableRegistry; @@ -298,7 +291,7 @@ public class SearchModule extends AbstractModule { this.settings = settings; this.namedWriteableRegistry = namedWriteableRegistry; suggesters = new Suggesters(namedWriteableRegistry); - + highlighters = new Highlighters(settings); registerBuiltinScoreFunctionParsers(); registerBuiltinQueryParsers(); registerBuiltinRescorers(); @@ -306,10 +299,11 @@ public class SearchModule extends AbstractModule { registerBuiltinValueFormats(); registerBuiltinSignificanceHeuristics(); registerBuiltinMovingAverageModels(); + registerBuiltinSubFetchPhases(); } - public void registerHighlighter(String key, Class clazz) { - highlighters.registerExtension(key, clazz); + public void registerHighlighter(String key, Highlighter highligher) { + highlighters.registerHighlighter(key, highligher); } public void registerSuggester(String key, Suggester suggester) { @@ -331,13 +325,6 @@ public class SearchModule extends AbstractModule { namedWriteableRegistry.register(ScoreFunctionBuilder.class, functionName.getPreferredName(), reader); } - /** - * Fetch the registry of {@linkplain ScoreFunction}s. This is public so extensions can access the score functions. - */ - public ParseFieldRegistry> getScoreFunctionParserRegistry() { - return scoreFunctionParserRegistry; - } - /** * Register a new ValueFormat. */ @@ -367,8 +354,19 @@ public class SearchModule extends AbstractModule { return queryParserRegistry; } - public void registerFetchSubPhase(Class subPhase) { - fetchSubPhases.add(subPhase); + /** + * Registers a {@link FetchSubPhase} instance. This sub phase is executed when docuemnts are fetched for instanced to highlight + * documents. + */ + public void registerFetchSubPhase(FetchSubPhase subPhase) { + fetchSubPhases.add(Objects.requireNonNull(subPhase, "FetchSubPhase must not be null")); + } + + /** + * Returns the {@link Highlighter} registry + */ + public Highlighters getHighlighters() { + return highlighters; } /** @@ -441,41 +439,15 @@ public class SearchModule extends AbstractModule { namedWriteableRegistry.register(PipelineAggregatorBuilder.class, aggregationName.getPreferredName(), reader); } - public AggregatorParsers getAggregatorParsers() { - return aggregatorParsers; - } - @Override protected void configure() { bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry); bind(Suggesters.class).toInstance(suggesters); configureSearch(); configureAggs(); - configureHighlighters(); - configureFetchSubPhase(); configureShapes(); } - protected void configureFetchSubPhase() { - Multibinder fetchSubPhaseMultibinder = Multibinder.newSetBinder(binder(), FetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(ExplainFetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(FieldDataFieldsFetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(ScriptFieldsFetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(FetchSourceSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(VersionFetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(MatchedQueriesFetchSubPhase.class); - fetchSubPhaseMultibinder.addBinding().to(HighlightPhase.class); - fetchSubPhaseMultibinder.addBinding().to(ParentFieldSubFetchPhase.class); - for (Class clazz : fetchSubPhases) { - fetchSubPhaseMultibinder.addBinding().to(clazz); - } - bind(InnerHitsFetchSubPhase.class).asEagerSingleton(); - } - - protected void configureHighlighters() { - highlighters.bind(binder()); - } - protected void configureAggs() { registerAggregation(AvgAggregationBuilder::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD); registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD); @@ -555,18 +527,13 @@ public class SearchModule extends AbstractModule { BucketSelectorPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD); registerPipelineAggregation(SerialDiffPipelineAggregatorBuilder::new, SerialDiffPipelineAggregatorBuilder::parse, SerialDiffPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD); - - AggregationPhase aggPhase = new AggregationPhase(); bind(AggregatorParsers.class).toInstance(aggregatorParsers); - bind(AggregationPhase.class).toInstance(aggPhase); } protected void configureSearch() { // configure search private classes... - bind(DfsPhase.class).asEagerSingleton(); - bind(QueryPhase.class).asEagerSingleton(); bind(SearchPhaseController.class).asEagerSingleton(); - bind(FetchPhase.class).asEagerSingleton(); + bind(FetchPhase.class).toInstance(new FetchPhase(fetchSubPhases)); bind(SearchTransportService.class).asEagerSingleton(); if (searchServiceImpl == SearchService.class) { bind(SearchService.class).asEagerSingleton(); @@ -637,6 +604,17 @@ public class SearchModule extends AbstractModule { registerMovingAverageModel(HoltWintersModel.NAME_FIELD, HoltWintersModel::new, HoltWintersModel.PARSER); } + private void registerBuiltinSubFetchPhases() { + registerFetchSubPhase(new ExplainFetchSubPhase()); + registerFetchSubPhase(new FieldDataFieldsFetchSubPhase()); + registerFetchSubPhase(new ScriptFieldsFetchSubPhase()); + registerFetchSubPhase(new FetchSourceSubPhase()); + registerFetchSubPhase(new VersionFetchSubPhase()); + registerFetchSubPhase(new MatchedQueriesFetchSubPhase()); + registerFetchSubPhase(new HighlightPhase(settings, highlighters)); + registerFetchSubPhase(new ParentFieldSubFetchPhase()); + } + private void registerBuiltinQueryParsers() { registerQuery(MatchQueryBuilder::new, MatchQueryBuilder::fromXContent, MatchQueryBuilder.QUERY_NAME_FIELD); registerQuery(MatchPhraseQueryBuilder::new, MatchPhraseQueryBuilder::fromXContent, MatchPhraseQueryBuilder.QUERY_NAME_FIELD); diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 57841466a62..dfbde3a6b19 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -134,7 +134,7 @@ public class SearchService extends AbstractLifecycleComponent imp private final BigArrays bigArrays; - private final DfsPhase dfsPhase; + private final DfsPhase dfsPhase = new DfsPhase(); private final QueryPhase queryPhase; @@ -158,8 +158,8 @@ public class SearchService extends AbstractLifecycleComponent imp @Inject public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService, - ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, DfsPhase dfsPhase, - QueryPhase queryPhase, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) { + ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, + FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) { super(settings); this.aggParsers = aggParsers; this.suggesters = suggesters; @@ -169,8 +169,7 @@ public class SearchService extends AbstractLifecycleComponent imp this.indicesService = indicesService; this.scriptService = scriptService; this.bigArrays = bigArrays; - this.dfsPhase = dfsPhase; - this.queryPhase = queryPhase; + this.queryPhase = new QueryPhase(settings); this.fetchPhase = fetchPhase; TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings); @@ -257,8 +256,7 @@ public class SearchService extends AbstractLifecycleComponent imp /** * Try to load the query results from the cache or execute the query phase directly if the cache cannot be used. */ - private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context, - final QueryPhase queryPhase) throws Exception { + private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context) throws Exception { final boolean canCache = indicesService.canCache(request, context); if (canCache) { indicesService.loadIntoContext(request, context, queryPhase); @@ -275,7 +273,7 @@ public class SearchService extends AbstractLifecycleComponent imp long time = System.nanoTime(); contextProcessing(context); - loadOrExecuteQueryPhase(request, context, queryPhase); + loadOrExecuteQueryPhase(request, context); if (context.queryResult().topDocs().scoreDocs.length == 0 && context.scrollContext() == null) { freeContext(context.id()); @@ -367,7 +365,7 @@ public class SearchService extends AbstractLifecycleComponent imp operationListener.onPreQueryPhase(context); long time = System.nanoTime(); try { - loadOrExecuteQueryPhase(request, context, queryPhase); + loadOrExecuteQueryPhase(request, context); } catch (Throwable e) { operationListener.onFailedQueryPhase(context); throw ExceptionsHelper.convertToRuntime(e); @@ -826,9 +824,7 @@ public class SearchService extends AbstractLifecycleComponent imp if (context.scrollContext() == null) { throw new SearchContextException(context, "`slice` cannot be used outside of a scroll context"); } - context.sliceFilter(source.slice().toFilter(queryShardContext, - context.shardTarget().getShardId().getId(), - queryShardContext.getIndexSettings().getNumberOfShards())); + context.sliceBuilder(source.slice()); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java index 38fb0e6a431..2d8ea318fd6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java @@ -27,6 +27,8 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement; +import org.elasticsearch.search.profile.Profilers; +import org.elasticsearch.search.profile.aggregation.ProfilingAggregator; import java.io.IOException; import java.util.ArrayList; @@ -81,7 +83,12 @@ public class AggregatorFactories { // propagate the fact that only bucket 0 will be collected with single-bucket // aggs final boolean collectsFromSingleBucket = false; - aggregators[i] = factories[i].create(parent, collectsFromSingleBucket); + Aggregator factory = factories[i].create(parent, collectsFromSingleBucket); + Profilers profilers = factory.context().searchContext().getProfilers(); + if (profilers != null) { + factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler()); + } + aggregators[i] = factory; } return aggregators; } @@ -92,7 +99,12 @@ public class AggregatorFactories { for (int i = 0; i < factories.length; i++) { // top-level aggs only get called with bucket 0 final boolean collectsFromSingleBucket = true; - aggregators[i] = factories[i].create(null, collectsFromSingleBucket); + Aggregator factory = factories[i].create(null, collectsFromSingleBucket); + Profilers profilers = factory.context().searchContext().getProfilers(); + if (profilers != null) { + factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler()); + } + aggregators[i] = factory; } return aggregators; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java index 3fced89a014..854838b7441 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java @@ -28,13 +28,139 @@ import org.elasticsearch.search.aggregations.InternalAggregation.Type; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.internal.SearchContext.Lifetime; - import java.io.IOException; import java.util.List; import java.util.Map; public abstract class AggregatorFactory> { + public static final class MultiBucketAggregatorWrapper extends Aggregator { + private final BigArrays bigArrays; + private final Aggregator parent; + private final AggregatorFactory factory; + private final Aggregator first; + ObjectArray aggregators; + ObjectArray collectors; + + MultiBucketAggregatorWrapper(BigArrays bigArrays, AggregationContext context, Aggregator parent, AggregatorFactory factory, + Aggregator first) { + this.bigArrays = bigArrays; + this.parent = parent; + this.factory = factory; + this.first = first; + context.searchContext().addReleasable(this, Lifetime.PHASE); + aggregators = bigArrays.newObjectArray(1); + aggregators.set(0, first); + collectors = bigArrays.newObjectArray(1); + } + + public Class getWrappedClass() { + return first.getClass(); + } + + @Override + public String name() { + return first.name(); + } + + @Override + public AggregationContext context() { + return first.context(); + } + + @Override + public Aggregator parent() { + return first.parent(); + } + + @Override + public boolean needsScores() { + return first.needsScores(); + } + + @Override + public Aggregator subAggregator(String name) { + throw new UnsupportedOperationException(); + } + + @Override + public void preCollection() throws IOException { + for (long i = 0; i < aggregators.size(); ++i) { + final Aggregator aggregator = aggregators.get(i); + if (aggregator != null) { + aggregator.preCollection(); + } + } + } + + @Override + public void postCollection() throws IOException { + for (long i = 0; i < aggregators.size(); ++i) { + final Aggregator aggregator = aggregators.get(i); + if (aggregator != null) { + aggregator.postCollection(); + } + } + } + + @Override + public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx) { + for (long i = 0; i < collectors.size(); ++i) { + collectors.set(i, null); + } + return new LeafBucketCollector() { + Scorer scorer; + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public void collect(int doc, long bucket) throws IOException { + collectors = bigArrays.grow(collectors, bucket + 1); + + LeafBucketCollector collector = collectors.get(bucket); + if (collector == null) { + aggregators = bigArrays.grow(aggregators, bucket + 1); + Aggregator aggregator = aggregators.get(bucket); + if (aggregator == null) { + aggregator = factory.create(parent, true); + aggregator.preCollection(); + aggregators.set(bucket, aggregator); + } + collector = aggregator.getLeafCollector(ctx); + collector.setScorer(scorer); + collectors.set(bucket, collector); + } + collector.collect(doc, 0); + } + + }; + } + + @Override + public InternalAggregation buildAggregation(long bucket) throws IOException { + if (bucket < aggregators.size()) { + Aggregator aggregator = aggregators.get(bucket); + if (aggregator != null) { + return aggregator.buildAggregation(0); + } + } + return buildEmptyAggregation(); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return first.buildEmptyAggregation(); + } + + @Override + public void close() { + Releasables.close(aggregators, collectors); + } + } + protected final String name; protected final Type type; protected final AggregatorFactory parent; @@ -112,120 +238,7 @@ public abstract class AggregatorFactory> { final Aggregator parent) throws IOException { final Aggregator first = factory.create(parent, true); final BigArrays bigArrays = context.bigArrays(); - return new Aggregator() { - - ObjectArray aggregators; - ObjectArray collectors; - - { - context.searchContext().addReleasable(this, Lifetime.PHASE); - aggregators = bigArrays.newObjectArray(1); - aggregators.set(0, first); - collectors = bigArrays.newObjectArray(1); - } - - @Override - public String name() { - return first.name(); - } - - @Override - public AggregationContext context() { - return first.context(); - } - - @Override - public Aggregator parent() { - return first.parent(); - } - - @Override - public boolean needsScores() { - return first.needsScores(); - } - - @Override - public Aggregator subAggregator(String name) { - throw new UnsupportedOperationException(); - } - - @Override - public void preCollection() throws IOException { - for (long i = 0; i < aggregators.size(); ++i) { - final Aggregator aggregator = aggregators.get(i); - if (aggregator != null) { - aggregator.preCollection(); - } - } - } - - @Override - public void postCollection() throws IOException { - for (long i = 0; i < aggregators.size(); ++i) { - final Aggregator aggregator = aggregators.get(i); - if (aggregator != null) { - aggregator.postCollection(); - } - } - } - - @Override - public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx) { - for (long i = 0; i < collectors.size(); ++i) { - collectors.set(i, null); - } - return new LeafBucketCollector() { - Scorer scorer; - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - - @Override - public void collect(int doc, long bucket) throws IOException { - aggregators = bigArrays.grow(aggregators, bucket + 1); - collectors = bigArrays.grow(collectors, bucket + 1); - - LeafBucketCollector collector = collectors.get(bucket); - if (collector == null) { - Aggregator aggregator = aggregators.get(bucket); - if (aggregator == null) { - aggregator = factory.create(parent, true); - aggregator.preCollection(); - aggregators.set(bucket, aggregator); - } - collector = aggregator.getLeafCollector(ctx); - collector.setScorer(scorer); - collectors.set(bucket, collector); - } - collector.collect(doc, 0); - } - - }; - } - - @Override - public InternalAggregation buildAggregation(long bucket) throws IOException { - if (bucket < aggregators.size()) { - Aggregator aggregator = aggregators.get(bucket); - if (aggregator != null) { - return aggregator.buildAggregation(0); - } - } - return buildEmptyAggregation(); - } - - @Override - public InternalAggregation buildEmptyAggregation() { - return first.buildEmptyAggregation(); - } - - @Override - public void close() { - Releasables.close(aggregators, collectors); - } - }; + return new MultiBucketAggregatorWrapper(bigArrays, context, parent, factory, first); } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 68e93f41991..4f00a49a1d9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; @@ -84,9 +85,9 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { int shardSize = this.shardSize; - if (shardSize == 0) { - shardSize = Integer.MAX_VALUE; - } int requiredSize = this.requiredSize; - if (requiredSize == 0) { - requiredSize = Integer.MAX_VALUE; - } if (shardSize < 0) { - // Use default heuristic to avoid any wrong-ranking caused by distributed counting + // Use default heuristic to avoid any wrong-ranking caused by + // distributed counting shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, context.searchContext().numberOfShards()); } + if (requiredSize <= 0 || shardSize <= 0) { + throw new ElasticsearchException( + "parameters [required_size] and [shard_size] must be >0 in geohash_grid aggregation [" + name + "]."); + } + if (shardSize < requiredSize) { shardSize = requiredSize; } @@ -139,7 +140,9 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder -1) { + builder.field(GeoHashGridParams.FIELD_SHARD_SIZE.getPreferredName(), shardSize); + } return builder; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java index 64a9c607729..e3886bee563 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java @@ -113,8 +113,8 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB * (defaults to 10) */ public SignificantTermsAggregationBuilder size(int size) { - if (size < 0) { - throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]"); + if (size <= 0) { + throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]"); } bucketCountThresholds.setRequiredSize(size); return this; @@ -127,9 +127,9 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB * results are. */ public SignificantTermsAggregationBuilder shardSize(int shardSize) { - if (shardSize < 0) { + if (shardSize <= 0) { throw new IllegalArgumentException( - "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]"); + "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]"); } bucketCountThresholds.setShardSize(shardSize); return this; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index f4cb133c499..1dd59edf6d9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -110,8 +110,8 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder entries from every shards in order to return if (shardSize < requiredSize) { setShardSize(requiredSize); @@ -100,8 +93,12 @@ public abstract class TermsAggregator extends BucketsAggregator { setShardMinDocCount(minDocCount); } - if (requiredSize < 0 || minDocCount < 0) { - throw new ElasticsearchException("parameters [requiredSize] and [minDocCount] must be >=0 in terms aggregation."); + if (requiredSize <= 0 || shardSize <= 0) { + throw new ElasticsearchException("parameters [required_size] and [shard_size] must be >0 in terms aggregation."); + } + + if (minDocCount < 0 || shardMinDocCount < 0) { + throw new ElasticsearchException("parameter [min_doc_count] and [shardMinDocCount] must be >=0 in terms aggregation."); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index b5ed80a0022..0793bacf722 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -35,7 +35,6 @@ import org.elasticsearch.common.util.LongArray; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java index 192ad6c28dc..ec838e7dd41 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java @@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; @@ -82,9 +82,9 @@ public final class GeoCentroidAggregator extends MetricsAggregator { counts.increment(bucket, valueCount); // get the previous GeoPoint if a moving avg was computed if (prevCounts > 0) { - final GeoPoint centroid = GeoPoint.fromIndexLong(centroids.get(bucket)); - pt[0] = centroid.lon(); - pt[1] = centroid.lat(); + final long mortonCode = centroids.get(bucket); + pt[0] = GeoPointField.decodeLongitude(mortonCode); + pt[1] = GeoPointField.decodeLatitude(mortonCode); } // update the moving average for (int i = 0; i < valueCount; ++i) { @@ -92,7 +92,9 @@ public final class GeoCentroidAggregator extends MetricsAggregator { pt[0] = pt[0] + (value.getLon() - pt[0]) / ++prevCounts; pt[1] = pt[1] + (value.getLat() - pt[1]) / prevCounts; } - centroids.set(bucket, GeoEncodingUtils.mortonHash(pt[1], pt[0])); + // TODO: we do not need to interleave the lat and lon bits here + // should we just store them contiguously? + centroids.set(bucket, GeoPointField.encodeLatLon(pt[1], pt[0])); } } }; @@ -104,8 +106,10 @@ public final class GeoCentroidAggregator extends MetricsAggregator { return buildEmptyAggregation(); } final long bucketCount = counts.get(bucket); - final GeoPoint bucketCentroid = (bucketCount > 0) ? GeoPoint.fromIndexLong(centroids.get(bucket)) : - new GeoPoint(Double.NaN, Double.NaN); + final long mortonCode = centroids.get(bucket); + final GeoPoint bucketCentroid = (bucketCount > 0) + ? new GeoPoint(GeoPointField.decodeLatitude(mortonCode), GeoPointField.decodeLongitude(mortonCode)) + : null; return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java index 2798169b699..2bb3056ca66 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; -import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -61,6 +61,7 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G public InternalGeoCentroid(String name, GeoPoint centroid, long count, List pipelineAggregators, Map metaData) { super(name, pipelineAggregators, metaData); + assert (centroid == null) == (count == 0); this.centroid = centroid; assert count >= 0; this.count = count; @@ -68,7 +69,7 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G @Override public GeoPoint centroid() { - return (centroid == null || Double.isNaN(centroid.lon()) ? null : centroid); + return centroid; } @Override @@ -128,7 +129,8 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G protected void doReadFrom(StreamInput in) throws IOException { count = in.readVLong(); if (in.readBoolean()) { - centroid = GeoPoint.fromIndexLong(in.readLong()); + final long hash = in.readLong(); + centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); } else { centroid = null; } @@ -139,7 +141,8 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G out.writeVLong(count); if (centroid != null) { out.writeBoolean(true); - out.writeLong(GeoEncodingUtils.mortonHash(centroid.lat(), centroid.lon())); + // should we just write lat and lon separately? + out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon())); } else { out.writeBoolean(false); } diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java index 36fe562a568..b2ce044e4fc 100644 --- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java +++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java @@ -51,8 +51,9 @@ import org.elasticsearch.search.fetch.FetchSearchResultProvider; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; -import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.profile.ProfileShardResult; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.profile.query.QueryProfileShardResult; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.query.QuerySearchResultProvider; import org.elasticsearch.search.suggest.Suggest; @@ -407,7 +408,7 @@ public class SearchPhaseController extends AbstractComponent { //Collect profile results SearchProfileShardResults shardResults = null; if (!queryResults.isEmpty() && firstResult.profileResults() != null) { - Map> profileResults = new HashMap<>(queryResults.size()); + Map profileResults = new HashMap<>(queryResults.size()); for (AtomicArray.Entry entry : queryResults) { String key = entry.value.queryResult().shardTarget().toString(); profileResults.put(key, entry.value.queryResult().profileResults()); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 1b08df76743..e884d32f3e4 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -29,7 +29,6 @@ import org.apache.lucene.util.BitSet; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.text.Text; @@ -73,11 +72,9 @@ public class FetchPhase implements SearchPhase { private final FetchSubPhase[] fetchSubPhases; - @Inject - public FetchPhase(Set fetchSubPhases, InnerHitsFetchSubPhase innerHitsFetchSubPhase) { - innerHitsFetchSubPhase.setFetchPhase(this); + public FetchPhase(Set fetchSubPhases) { this.fetchSubPhases = fetchSubPhases.toArray(new FetchSubPhase[fetchSubPhases.size() + 1]); - this.fetchSubPhases[fetchSubPhases.size()] = innerHitsFetchSubPhase; + this.fetchSubPhases[fetchSubPhases.size()] = new InnerHitsFetchSubPhase(this); } @Override @@ -163,16 +160,12 @@ public class FetchPhase implements SearchPhase { hits[index] = searchHit; hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - if (fetchSubPhase.hitExecutionNeeded(context)) { - fetchSubPhase.hitExecute(context, hitContext); - } + fetchSubPhase.hitExecute(context, hitContext); } } for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - if (fetchSubPhase.hitsExecutionNeeded(context)) { - fetchSubPhase.hitsExecute(context, hits); - } + fetchSubPhase.hitsExecute(context, hits); } context.fetchResult().hits(new InternalSearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore())); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 5ba75b84fcc..b55afe642ec 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -36,7 +36,7 @@ import java.util.Map; */ public interface FetchSubPhase { - public static class HitContext { + class HitContext { private InternalSearchHit hit; private IndexSearcher searcher; private LeafReaderContext readerContext; @@ -87,16 +87,13 @@ public interface FetchSubPhase { return Collections.emptyMap(); } - boolean hitExecutionNeeded(SearchContext context); - /** * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc). */ - void hitExecute(SearchContext context, HitContext hitContext); + default void hitExecute(SearchContext context, HitContext hitContext) {} - boolean hitsExecutionNeeded(SearchContext context); - void hitsExecute(SearchContext context, InternalSearchHit[] hits); + default void hitsExecute(SearchContext context, InternalSearchHit[] hits) {} /** * This interface is in the fetch phase plugin mechanism. @@ -104,16 +101,16 @@ public interface FetchSubPhase { * Fetch phases that use the plugin mechanism must provide a ContextFactory to the SearchContext that creates the fetch phase context and also associates them with a name. * See {@link SearchContext#getFetchSubPhaseContext(FetchSubPhase.ContextFactory)} */ - public interface ContextFactory { + interface ContextFactory { /** * The name of the context. */ - public String getName(); + String getName(); /** * Creates a new instance of a FetchSubPhaseContext that holds all information a FetchSubPhase needs to execute on hits. */ - public SubPhaseContext newContextInstance(); + SubPhaseContext newContextInstance(); } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java index 0d8b4ef3271..e560b815d5a 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java @@ -30,24 +30,13 @@ import java.io.IOException; /** * */ -public class ExplainFetchSubPhase implements FetchSubPhase { - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.explain(); - } +public final class ExplainFetchSubPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.explain() == false) { + return; + } try { final int topLevelDocId = hitContext.hit().docId(); Explanation explanation = context.searcher().explain(context.query(), topLevelDocId); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java index f8034719a1d..81907d0b906 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java @@ -18,13 +18,11 @@ */ package org.elasticsearch.search.fetch.fielddata; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHitField; import org.elasticsearch.search.internal.SearchContext; @@ -37,7 +35,7 @@ import java.util.HashMap; * * Specifying {@code "fielddata_fields": ["field1", "field2"]} */ -public class FieldDataFieldsFetchSubPhase implements FetchSubPhase { +public final class FieldDataFieldsFetchSubPhase implements FetchSubPhase { public static final String[] NAMES = {"fielddata_fields", "fielddataFields"}; public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { @@ -53,29 +51,14 @@ public class FieldDataFieldsFetchSubPhase implements FetchSubPhase { } }; - @Inject - public FieldDataFieldsFetchSubPhase() { - } - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded(); - } - @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) { + return; + } for (FieldDataFieldsContext.FieldDataField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) { if (hitContext.hit().fieldsOrNull() == null) { - hitContext.hit().fields(new HashMap(2)); + hitContext.hit().fields(new HashMap<>(2)); } SearchHitField hitField = hitContext.hit().fields().get(field.name()); if (hitField == null) { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java index 31921457207..f34da5301d5 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java @@ -175,7 +175,7 @@ public final class InnerHitsContext { @Override public boolean equals(Object obj) { - if (super.equals(obj) == false) { + if (sameClassAs(obj) == false) { return false; } NestedChildrenQuery other = (NestedChildrenQuery) obj; @@ -187,7 +187,7 @@ public final class InnerHitsContext { @Override public int hashCode() { - int hash = super.hashCode(); + int hash = classHash(); hash = 31 * hash + parentFilter.hashCode(); hash = 31 * hash + childFilter.hashCode(); hash = 31 * hash + docId; diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java index 993d1ccbb5b..35a25b522c0 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java @@ -23,8 +23,6 @@ import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; @@ -33,34 +31,22 @@ import org.elasticsearch.search.internal.InternalSearchHits; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -/** - */ -public class InnerHitsFetchSubPhase implements FetchSubPhase { +public final class InnerHitsFetchSubPhase implements FetchSubPhase { - private FetchPhase fetchPhase; + private final FetchPhase fetchPhase; - @Inject - public InnerHitsFetchSubPhase() { - } - - @Override - public Map parseElements() { - // SearchParse elements needed because everything is parsed by InnerHitBuilder and eventually put - // into the search context. - return Collections.emptyMap(); - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.innerHits() != null && context.innerHits().getInnerHits().size() > 0; + public InnerHitsFetchSubPhase(FetchPhase fetchPhase) { + this.fetchPhase = fetchPhase; } @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if ((context.innerHits() != null && context.innerHits().getInnerHits().size() > 0) == false) { + return; + } Map results = new HashMap<>(); for (Map.Entry entry : context.innerHits().getInnerHits().entrySet()) { InnerHitsContext.BaseInnerHits innerHits = entry.getValue(); @@ -93,18 +79,4 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase { } hitContext.hit().setInnerHits(results); } - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - // To get around cyclic dependency issue - public void setFetchPhase(FetchPhase fetchPhase) { - this.fetchPhase = fetchPhase; - } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java index 983e131215d..59225f93a61 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.fetch.matchedqueries; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.search.Query; @@ -26,7 +27,6 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; @@ -39,22 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static java.util.Collections.emptyMap; - -/** - * - */ -public class MatchedQueriesFetchSubPhase implements FetchSubPhase { - - @Override - public Map parseElements() { - return emptyMap(); - } - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return true; // we short-circuit in hitsExecute - } +public final class MatchedQueriesFetchSubPhase implements FetchSubPhase { @Override public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { @@ -82,12 +67,13 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase { int docBase = -1; Weight weight = context.searcher().createNormalizedWeight(query, false); Bits matchingDocs = null; + final IndexReader indexReader = context.searcher().getIndexReader(); for (int i = 0; i < hits.length; ++i) { InternalSearchHit hit = hits[i]; - int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); + int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), indexReader.leaves()); if (readerIndex != hitReaderIndex) { readerIndex = hitReaderIndex; - LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(readerIndex); + LeafReaderContext ctx = indexReader.leaves().get(readerIndex); docBase = ctx.docBase; // scorers can be costly to create, so reuse them across docs of the same segment Scorer scorer = weight.scorer(ctx); @@ -99,7 +85,7 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase { } } for (int i = 0; i < hits.length; ++i) { - hits[i].matchedQueries(matchedQueries[i].toArray(new String[0])); + hits[i].matchedQueries(matchedQueries[i].toArray(new String[matchedQueries[i].size()])); } } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); @@ -107,14 +93,4 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase { SearchContext.current().clearReleasables(Lifetime.COLLECTION); } } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitExecute(SearchContext context, HitContext hitContext) { - // we do everything in hitsExecute - } } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java index 41fe7176424..6ace9a86a3e 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java @@ -25,10 +25,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHitField; import org.elasticsearch.search.internal.SearchContext; @@ -37,17 +34,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -public class ParentFieldSubFetchPhase implements FetchSubPhase { - - @Override - public Map parseElements() { - return Collections.emptyMap(); - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return true; - } +public final class ParentFieldSubFetchPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) { @@ -65,15 +52,6 @@ public class ParentFieldSubFetchPhase implements FetchSubPhase { fields.put(ParentFieldMapper.NAME, new InternalSearchHitField(ParentFieldMapper.NAME, Collections.singletonList(parentId))); } - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) { try { SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name()); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java index 540ce9aa66b..0838467bb6c 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java @@ -21,7 +21,6 @@ package org.elasticsearch.search.fetch.script; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.InternalSearchHitField; import org.elasticsearch.search.internal.SearchContext; @@ -32,27 +31,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -/** - * - */ -public class ScriptFieldsFetchSubPhase implements FetchSubPhase { - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.hasScriptFields(); - } +public final class ScriptFieldsFetchSubPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.hasScriptFields() == false) { + return; + } for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) { LeafSearchScript leafScript; try { @@ -62,10 +47,9 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase { } leafScript.setDocument(hitContext.docId()); - Object value; + final Object value; try { - value = leafScript.run(); - value = leafScript.unwrap(value); + value = leafScript.unwrap(leafScript.run()); } catch (RuntimeException e) { if (scriptField.ignoreException()) { continue; @@ -74,7 +58,7 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase { } if (hitContext.hit().fieldsOrNull() == null) { - hitContext.hit().fields(new HashMap(2)); + hitContext.hit().fields(new HashMap<>(2)); } SearchHitField hitField = hitContext.hit().fields().get(scriptField.name()); @@ -84,7 +68,7 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase { values = Collections.emptyList(); } else if (value instanceof Collection) { // TODO: use diamond operator once JI-9019884 is fixed - values = new ArrayList((Collection) value); + values = new ArrayList<>((Collection) value); } else { values = Collections.singletonList(value); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java index 30220d12694..900402bda13 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java @@ -23,32 +23,18 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; -/** - */ -public class FetchSourceSubPhase implements FetchSubPhase { - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.sourceRequested(); - } +public final class FetchSourceSubPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.sourceRequested() == false) { + return; + } FetchSourceContext fetchSourceContext = context.fetchSourceContext(); assert fetchSourceContext.fetchSource(); if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java index c5511352ec2..77a0e954b2d 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java @@ -25,36 +25,21 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; -/** - * - */ -public class VersionFetchSubPhase implements FetchSubPhase { - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.version(); - } +public final class VersionFetchSubPhase implements FetchSubPhase { @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.version() == false) { + return; + } // it might make sense to cache the TermDocs on a shared fetch context and just skip here) // it is going to mean we work on the high level multi reader and not the lower level reader as is // the case below... - long version; + final long version; try { BytesRef uid = Uid.createUidAsBytes(hitContext.hit().type(), hitContext.hit().id()); version = Versions.loadVersion( @@ -64,10 +49,6 @@ public class VersionFetchSubPhase implements FetchSubPhase { } catch (IOException e) { throw new ElasticsearchException("Could not query index for _version", e); } - - if (version < 0) { - version = -1; - } - hitContext.hit().version(version); + hitContext.hit().version(version < 0 ? -1 : version); } } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index 92011fd77e7..7952addbe8c 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -21,7 +21,6 @@ package org.elasticsearch.search.highlight; import org.apache.lucene.search.Query; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; @@ -30,9 +29,8 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.core.TextFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; -import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.Highlighters; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.search.internal.SearchContext; import java.util.Arrays; @@ -42,45 +40,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * - */ public class HighlightPhase extends AbstractComponent implements FetchSubPhase { private static final List STANDARD_HIGHLIGHTERS_BY_PRECEDENCE = Arrays.asList("fvh", "postings", "plain"); private final Highlighters highlighters; - @Inject public HighlightPhase(Settings settings, Highlighters highlighters) { super(settings); this.highlighters = highlighters; } - /** - * highlighters do not have a parse element, they use - * {@link HighlightBuilder#fromXContent(org.elasticsearch.index.query.QueryParseContext)} for parsing instead. - */ - @Override - public Map parseElements() { - return Collections.emptyMap(); - } - - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.highlight() != null; - } - @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.highlight() == null) { + return; + } Map highlightFields = new HashMap<>(); for (SearchContextHighlight.Field field : context.highlight().fields()) { Collection fieldNamesToHighlight; diff --git a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java b/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java deleted file mode 100644 index 30b8d15d93d..00000000000 --- a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.highlight; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.ExtensionPoint; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - -/** - * An extensions point and registry for all the highlighters a node supports. - */ -public class Highlighters extends ExtensionPoint.ClassMap { - - private static final String FVH = "fvh"; - private static final String PLAIN = "plain"; - private static final String POSTINGS = "postings"; - - private final Map parsers; - - public Highlighters(){ - this(Collections.emptyMap()); - } - - private Highlighters(Map parsers) { - super("highlighter", Highlighter.class, new HashSet<>(Arrays.asList(FVH, PLAIN, POSTINGS)), - Highlighters.class); - this.parsers = Collections.unmodifiableMap(parsers); - } - - @Inject - public Highlighters(Settings settings, Map parsers) { - this(addBuiltIns(settings, parsers)); - } - - private static Map addBuiltIns(Settings settings, Map parsers) { - Map map = new HashMap<>(); - map.put(FVH, new FastVectorHighlighter(settings)); - map.put(PLAIN, new PlainHighlighter()); - map.put(POSTINGS, new PostingsHighlighter()); - map.putAll(parsers); - return map; - } - - public Highlighter get(String type) { - return parsers.get(type); - } -} diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index df1007ebc71..50e91e082cd 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -122,7 +122,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { weight = super.createWeight(query, needsScores); } finally { profile.stopAndRecordTime(); - profiler.pollLastQuery(); + profiler.pollLastElement(); } return new ProfileWeight(query, weight, profile); } else { diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 30e994b7656..06df04db8a0 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -68,6 +68,7 @@ import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QueryPhaseExecutionException; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; +import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestionSearchContext; @@ -116,7 +117,7 @@ public class DefaultSearchContext extends SearchContext { private boolean trackScores = false; // when sorting, track scores as well... private FieldDoc searchAfter; // filter for sliced scroll - private Query sliceFilter; + private SliceBuilder sliceBuilder; /** * The original query as sent by the user without the types and aliases @@ -212,13 +213,23 @@ public class DefaultSearchContext extends SearchContext { if (rescoreContext.window() > maxWindow) { throw new QueryPhaseExecutionException(this, "Rescore window [" + rescoreContext.window() + "] is too large. It must " + "be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results to be " - + "rescored. This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + + "rescored. This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."); } } } + if (sliceBuilder != null) { + int sliceLimit = indexService.getIndexSettings().getMaxSlicesPerScroll(); + int numSlices = sliceBuilder.getMax(); + if (numSlices > sliceLimit) { + throw new QueryPhaseExecutionException(this, "The number of slices [" + numSlices + "] is too large. It must " + + "be less than [" + sliceLimit + "]. This limit can be set by changing the [" + + IndexSettings.MAX_SLICES_PER_SCROLL.getKey() + "] index level setting."); + } + } + // initialize the filtering alias based on the provided filters aliasFilter = indexService.aliasFilter(queryShardContext, request.filteringAliases()); @@ -257,9 +268,11 @@ public class DefaultSearchContext extends SearchContext { @Nullable public Query searchFilter(String[] types) { Query typesFilter = createSearchFilter(types, aliasFilter, mapperService().hasNested()); - if (sliceFilter == null) { + if (sliceBuilder == null) { return typesFilter; } + Query sliceFilter = sliceBuilder.toFilter(queryShardContext, shardTarget().getShardId().getId(), + queryShardContext.getIndexSettings().getNumberOfShards()); if (typesFilter == null) { return sliceFilter; } @@ -562,8 +575,8 @@ public class DefaultSearchContext extends SearchContext { return searchAfter; } - public SearchContext sliceFilter(Query filter) { - this.sliceFilter = filter; + public SearchContext sliceBuilder(SliceBuilder sliceBuilder) { + this.sliceBuilder = sliceBuilder; return this; } diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java index 9c33889dc9c..26410cc9680 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java +++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java @@ -28,13 +28,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.profile.ProfileShardResult; +import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.suggest.Suggest; import java.io.IOException; import java.util.Collections; -import java.util.List; import java.util.Map; import static org.elasticsearch.search.internal.InternalSearchHits.readSearchHits; @@ -99,7 +98,7 @@ public class InternalSearchResponse implements Streamable, ToXContent { * * @return Profile results */ - public Map> profile() { + public Map profile() { if (profileResults == null) { return Collections.emptyMap(); } diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java new file mode 100644 index 00000000000..31cb3c21237 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile; + +import org.elasticsearch.search.profile.query.QueryProfileBreakdown; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import java.util.Map; + +public abstract class AbstractInternalProfileTree, E> { + + protected ArrayList timings; + /** Maps the Query to it's list of children. This is basically the dependency tree */ + protected ArrayList> tree; + /** A list of the original queries, keyed by index position */ + protected ArrayList elements; + /** A list of top-level "roots". Each root can have its own tree of profiles */ + protected ArrayList roots; + /** A temporary stack used to record where we are in the dependency tree. */ + protected Deque stack; + private int currentToken = 0; + + public AbstractInternalProfileTree() { + timings = new ArrayList<>(10); + stack = new ArrayDeque<>(10); + tree = new ArrayList<>(10); + elements = new ArrayList<>(10); + roots = new ArrayList<>(10); + } + + /** + * Returns a {@link QueryProfileBreakdown} for a scoring query. Scoring queries (e.g. those + * that are past the rewrite phase and are now being wrapped by createWeight() ) follow + * a recursive progression. We can track the dependency tree by a simple stack + * + * The only hiccup is that the first scoring query will be identical to the last rewritten + * query, so we need to take special care to fix that + * + * @param query The scoring query we wish to profile + * @return A ProfileBreakdown for this query + */ + public PB getProfileBreakdown(E query) { + int token = currentToken; + + boolean stackEmpty = stack.isEmpty(); + + // If the stack is empty, we are a new root query + if (stackEmpty) { + + // We couldn't find a rewritten query to attach to, so just add it as a + // top-level root. This is just a precaution: it really shouldn't happen. + // We would only get here if a top-level query that never rewrites for some reason. + roots.add(token); + + // Increment the token since we are adding a new node, but notably, do not + // updateParent() because this was added as a root + currentToken += 1; + stack.add(token); + + return addDependencyNode(query, token); + } + + updateParent(token); + + // Increment the token since we are adding a new node + currentToken += 1; + stack.add(token); + + return addDependencyNode(query, token); + } + + /** + * Helper method to add a new node to the dependency tree. + * + * Initializes a new list in the dependency tree, saves the query and + * generates a new {@link QueryProfileBreakdown} to track the timings of + * this query + * + * @param element + * The element to profile + * @param token + * The assigned token for this element + * @return A ProfileBreakdown to profile this element + */ + private PB addDependencyNode(E element, int token) { + + // Add a new slot in the dependency tree + tree.add(new ArrayList<>(5)); + + // Save our query for lookup later + elements.add(element); + + PB queryTimings = createProfileBreakdown(); + timings.add(token, queryTimings); + return queryTimings; + } + + protected abstract PB createProfileBreakdown(); + + /** + * Removes the last (e.g. most recent) value on the stack + */ + public void pollLast() { + stack.pollLast(); + } + + /** + * After the query has been run and profiled, we need to merge the flat timing map + * with the dependency graph to build a data structure that mirrors the original + * query tree + * + * @return a hierarchical representation of the profiled query tree + */ + public List getTree() { + ArrayList results = new ArrayList<>(5); + for (Integer root : roots) { + results.add(doGetTree(root)); + } + return results; + } + + /** + * Recursive helper to finalize a node in the dependency tree + * @param token The node we are currently finalizing + * @return A hierarchical representation of the tree inclusive of children at this level + */ + private ProfileResult doGetTree(int token) { + E element = elements.get(token); + PB breakdown = timings.get(token); + Map timings = breakdown.toTimingMap(); + List children = tree.get(token); + List childrenProfileResults = Collections.emptyList(); + + if (children != null) { + childrenProfileResults = new ArrayList<>(children.size()); + for (Integer child : children) { + ProfileResult childNode = doGetTree(child); + childrenProfileResults.add(childNode); + } + } + + // TODO this would be better done bottom-up instead of top-down to avoid + // calculating the same times over and over...but worth the effort? + long nodeTime = getNodeTime(timings, childrenProfileResults); + String type = getTypeFromElement(element); + String description = getDescriptionFromElement(element); + return new ProfileResult(type, description, timings, childrenProfileResults, nodeTime); + } + + protected abstract String getTypeFromElement(E element); + + protected abstract String getDescriptionFromElement(E element); + + /** + * Internal helper to add a child to the current parent node + * + * @param childToken The child to add to the current parent + */ + private void updateParent(int childToken) { + Integer parent = stack.peekLast(); + ArrayList parentNode = tree.get(parent); + parentNode.add(childToken); + tree.set(parent, parentNode); + } + + /** + * Internal helper to calculate the time of a node, inclusive of children + * + * @param timings + * A map of breakdown timing for the node + * @param children + * All children profile results at this node + * @return The total time at this node, inclusive of children + */ + private static long getNodeTime(Map timings, List children) { + long nodeTime = 0; + for (long time : timings.values()) { + nodeTime += time; + } + + // Then add up our children + for (ProfileResult child : children) { + nodeTime += getNodeTime(child.getTimeBreakdown(), child.getProfiledChildren()); + } + return nodeTime; + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java new file mode 100644 index 00000000000..a7ccb72785e --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile; + +import java.util.List; + +public class AbstractProfiler, E> { + + protected final AbstractInternalProfileTree profileTree; + + public AbstractProfiler(AbstractInternalProfileTree profileTree) { + this.profileTree = profileTree; + } + + /** + * Get the {@link AbstractProfileBreakdown} for the given element in the + * tree, potentially creating it if it did not exist. + */ + public PB getQueryBreakdown(E query) { + return profileTree.getProfileBreakdown(query); + } + + /** + * Removes the last (e.g. most recent) element on the stack. + */ + public void pollLastElement() { + profileTree.pollLast(); + } + + /** + * @return a hierarchical representation of the profiled tree + */ + public List getTree() { + return profileTree.getTree(); + } + +} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java index 9def3db7582..2a1fb0ba9b1 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java +++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java @@ -22,83 +22,50 @@ package org.elasticsearch.search.profile; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.profile.query.CollectorResult; +import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult; +import org.elasticsearch.search.profile.query.QueryProfileShardResult; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -/** - * A container class to hold the profile results for a single shard in the request. - * Contains a list of query profiles, a collector tree and a total rewrite tree. - */ -public final class ProfileShardResult implements Writeable, ToXContent { +public class ProfileShardResult implements Writeable { - private final List queryProfileResults; + private final List queryProfileResults; - private final CollectorResult profileCollector; + private final AggregationProfileShardResult aggProfileShardResult; - private final long rewriteTime; - - public ProfileShardResult(List queryProfileResults, long rewriteTime, - CollectorResult profileCollector) { - assert(profileCollector != null); - this.queryProfileResults = queryProfileResults; - this.profileCollector = profileCollector; - this.rewriteTime = rewriteTime; + public ProfileShardResult(List queryProfileResults, AggregationProfileShardResult aggProfileShardResult) { + this.aggProfileShardResult = aggProfileShardResult; + this.queryProfileResults = Collections.unmodifiableList(queryProfileResults); } - /** - * Read from a stream. - */ public ProfileShardResult(StreamInput in) throws IOException { int profileSize = in.readVInt(); - queryProfileResults = new ArrayList<>(profileSize); - for (int j = 0; j < profileSize; j++) { - queryProfileResults.add(new ProfileResult(in)); + List queryProfileResults = new ArrayList<>(profileSize); + for (int i = 0; i < profileSize; i++) { + QueryProfileShardResult result = new QueryProfileShardResult(in); + queryProfileResults.add(result); } - - profileCollector = new CollectorResult(in); - rewriteTime = in.readLong(); + this.queryProfileResults = Collections.unmodifiableList(queryProfileResults); + this.aggProfileShardResult = new AggregationProfileShardResult(in); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(queryProfileResults.size()); - for (ProfileResult p : queryProfileResults) { - p.writeTo(out); + for (QueryProfileShardResult queryShardResult : queryProfileResults) { + queryShardResult.writeTo(out); } - profileCollector.writeTo(out); - out.writeLong(rewriteTime); + aggProfileShardResult.writeTo(out); } - - public List getQueryResults() { - return Collections.unmodifiableList(queryProfileResults); + public List getQueryProfileResults() { + return queryProfileResults; } - public long getRewriteTime() { - return rewriteTime; - } - - public CollectorResult getCollectorResult() { - return profileCollector; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray("query"); - for (ProfileResult p : queryProfileResults) { - p.toXContent(builder, params); - } - builder.endArray(); - builder.field("rewrite_time", rewriteTime); - builder.startArray("collector"); - profileCollector.toXContent(builder, params); - builder.endArray(); - return builder; + public AggregationProfileShardResult getAggregationProfileResults() { + return aggProfileShardResult; } } diff --git a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java index e9e6d88db18..d754be41f6d 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java +++ b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java @@ -20,22 +20,25 @@ package org.elasticsearch.search.profile; import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.profile.aggregation.AggregationProfiler; import org.elasticsearch.search.profile.query.QueryProfiler; import java.util.ArrayList; import java.util.Collections; import java.util.List; -/** Wrapper around several {@link QueryProfiler}s that makes management easier. */ +/** Wrapper around all the profilers that makes management easier. */ public final class Profilers { private final ContextIndexSearcher searcher; private final List queryProfilers; + private final AggregationProfiler aggProfiler; /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */ public Profilers(ContextIndexSearcher searcher) { this.searcher = searcher; this.queryProfilers = new ArrayList<>(); + this.aggProfiler = new AggregationProfiler(); addQueryProfiler(); } @@ -57,4 +60,9 @@ public final class Profilers { return Collections.unmodifiableList(queryProfilers); } + /** Return the {@link AggregationProfiler}. */ + public AggregationProfiler getAggregationProfiler() { + return aggProfiler; + } + } diff --git a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java index bf265dd9a7e..6794aa49399 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java +++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java @@ -24,6 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult; +import org.elasticsearch.search.profile.aggregation.AggregationProfiler; +import org.elasticsearch.search.profile.query.QueryProfileShardResult; import org.elasticsearch.search.profile.query.QueryProfiler; import java.io.IOException; @@ -32,7 +35,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * A container class to hold all the profile results across all shards. Internally @@ -40,17 +42,10 @@ import java.util.stream.Collectors; */ public final class SearchProfileShardResults implements Writeable, ToXContent{ - private Map> shardResults; + private Map shardResults; - public SearchProfileShardResults(Map> shardResults) { - Map> transformed = - shardResults.entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> Collections.unmodifiableList(e.getValue())) - ); - this.shardResults = Collections.unmodifiableMap(transformed); + public SearchProfileShardResults(Map shardResults) { + this.shardResults = Collections.unmodifiableMap(shardResults); } public SearchProfileShardResults(StreamInput in) throws IOException { @@ -59,33 +54,22 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{ for (int i = 0; i < size; i++) { String key = in.readString(); - int shardResultsSize = in.readInt(); - - List shardResult = new ArrayList<>(shardResultsSize); - - for (int j = 0; j < shardResultsSize; j++) { - ProfileShardResult result = new ProfileShardResult(in); - shardResult.add(result); - } - shardResults.put(key, Collections.unmodifiableList(shardResult)); + ProfileShardResult shardResult = new ProfileShardResult(in); + shardResults.put(key, shardResult); } shardResults = Collections.unmodifiableMap(shardResults); } - public Map> getShardResults() { + public Map getShardResults() { return this.shardResults; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeInt(shardResults.size()); - for (Map.Entry> entry : shardResults.entrySet()) { + for (Map.Entry entry : shardResults.entrySet()) { out.writeString(entry.getKey()); - out.writeInt(entry.getValue().size()); - - for (ProfileShardResult result : entry.getValue()) { - result.writeTo(out); - } + entry.getValue().writeTo(out); } } @@ -93,14 +77,18 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("profile").startArray("shards"); - for (Map.Entry> entry : shardResults.entrySet()) { - builder.startObject().field("id",entry.getKey()).startArray("searches"); - for (ProfileShardResult result : entry.getValue()) { + for (Map.Entry entry : shardResults.entrySet()) { + builder.startObject(); + builder.field("id", entry.getKey()); + builder.startArray("searches"); + for (QueryProfileShardResult result : entry.getValue().getQueryProfileResults()) { builder.startObject(); result.toXContent(builder, params); builder.endObject(); } - builder.endArray().endObject(); + builder.endArray(); + entry.getValue().getAggregationProfileResults().toXContent(builder, params); + builder.endObject(); } builder.endArray().endObject(); @@ -112,16 +100,20 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{ * can be serialized to other nodes, emitted as JSON, etc. * * @param profilers - * A list of Profilers to convert into - * InternalProfileShardResults - * @return A list of corresponding InternalProfileShardResults + * The {@link Profilers} to convert into results + * @return A {@link ProfileShardResult} representing the results for this + * shard */ - public static List buildShardResults(List profilers) { - List results = new ArrayList<>(profilers.size()); - for (QueryProfiler profiler : profilers) { - ProfileShardResult result = new ProfileShardResult(profiler.getQueryTree(), profiler.getRewriteTime(), profiler.getCollector()); - results.add(result); + public static ProfileShardResult buildShardResults(Profilers profilers) { + List queryProfilers = profilers.getQueryProfilers(); + AggregationProfiler aggProfiler = profilers.getAggregationProfiler(); + List queryResults = new ArrayList<>(queryProfilers.size()); + for (QueryProfiler queryProfiler : queryProfilers) { + QueryProfileShardResult result = new QueryProfileShardResult(queryProfiler.getTree(), queryProfiler.getRewriteTime(), + queryProfiler.getCollector()); + queryResults.add(result); } - return results; + AggregationProfileShardResult aggResults = new AggregationProfileShardResult(aggProfiler.getTree()); + return new ProfileShardResult(queryResults, aggResults); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java similarity index 64% rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java rename to core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java index a06bcdf9840..b4cb1efe5d3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java @@ -17,21 +17,14 @@ * under the License. */ -package org.elasticsearch.painless; +package org.elasticsearch.search.profile.aggregation; -import org.elasticsearch.painless.Variables.Reserved; -import org.elasticsearch.painless.node.SSource; +import org.elasticsearch.search.profile.AbstractProfileBreakdown; -/** - * Runs the analysis phase of compilation using the Painless AST. - */ -final class Analyzer { - static Variables analyze(Reserved shortcut, SSource root) { - Variables variables = new Variables(shortcut); - root.analyze(variables); +public class AggregationProfileBreakdown extends AbstractProfileBreakdown { - return variables; + public AggregationProfileBreakdown() { + super(AggregationTimingType.values()); } - private Analyzer() {} } diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java new file mode 100644 index 00000000000..df55c5592d6 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.profile.ProfileResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A container class to hold the profile results for a single shard in the request. + * Contains a list of query profiles, a collector tree and a total rewrite tree. + */ +public final class AggregationProfileShardResult implements Writeable, ToXContent { + + private final List aggProfileResults; + + public AggregationProfileShardResult(List aggProfileResults) { + this.aggProfileResults = aggProfileResults; + } + + /** + * Read from a stream. + */ + public AggregationProfileShardResult(StreamInput in) throws IOException { + int profileSize = in.readVInt(); + aggProfileResults = new ArrayList<>(profileSize); + for (int j = 0; j < profileSize; j++) { + aggProfileResults.add(new ProfileResult(in)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(aggProfileResults.size()); + for (ProfileResult p : aggProfileResults) { + p.writeTo(out); + } + } + + + public List getProfileResults() { + return Collections.unmodifiableList(aggProfileResults); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("aggregations"); + for (ProfileResult p : aggProfileResults) { + p.toXContent(builder, params); + } + builder.endArray(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java new file mode 100644 index 00000000000..45d401ccbdc --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.profile.AbstractProfiler; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class AggregationProfiler extends AbstractProfiler { + + private final Map, AggregationProfileBreakdown> profileBrakdownLookup = new HashMap<>(); + + public AggregationProfiler() { + super(new InternalAggregationProfileTree()); + } + + @Override + public AggregationProfileBreakdown getQueryBreakdown(Aggregator agg) { + List path = getAggregatorPath(agg); + AggregationProfileBreakdown aggregationProfileBreakdown = profileBrakdownLookup.get(path); + if (aggregationProfileBreakdown == null) { + aggregationProfileBreakdown = super.getQueryBreakdown(agg); + profileBrakdownLookup.put(path, aggregationProfileBreakdown); + } + return aggregationProfileBreakdown; + } + + public static List getAggregatorPath(Aggregator agg) { + LinkedList path = new LinkedList<>(); + while (agg != null) { + path.addFirst(agg.name()); + agg = agg.parent(); + } + return path; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java new file mode 100644 index 00000000000..d1c5d3dd538 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import java.util.Locale; + +public enum AggregationTimingType { + INITIALIZE, + COLLECT, + BUILD_AGGREGATION, + REDUCE; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java new file mode 100644 index 00000000000..f367595c84c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactory.MultiBucketAggregatorWrapper; +import org.elasticsearch.search.profile.AbstractInternalProfileTree; + +public class InternalAggregationProfileTree extends AbstractInternalProfileTree { + + @Override + protected AggregationProfileBreakdown createProfileBreakdown() { + return new AggregationProfileBreakdown(); + } + + @Override + protected String getTypeFromElement(Aggregator element) { + if (element instanceof MultiBucketAggregatorWrapper) { + return ((MultiBucketAggregatorWrapper) element).getWrappedClass().getName(); + } + return element.getClass().getName(); + } + + @Override + protected String getDescriptionFromElement(Aggregator element) { + return element.name(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java new file mode 100644 index 00000000000..2883c2903e8 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.support.AggregationContext; + +import java.io.IOException; + +public class ProfilingAggregator extends Aggregator { + + private final Aggregator delegate; + private final AggregationProfiler profiler; + private AggregationProfileBreakdown profileBreakdown; + + public ProfilingAggregator(Aggregator delegate, AggregationProfiler profiler) throws IOException { + this.profiler = profiler; + this.delegate = delegate; + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public boolean needsScores() { + return delegate.needsScores(); + } + + @Override + public String name() { + return delegate.name(); + } + + @Override + public AggregationContext context() { + return delegate.context(); + } + + @Override + public Aggregator parent() { + return delegate.parent(); + } + + @Override + public Aggregator subAggregator(String name) { + return delegate.subAggregator(name); + } + + @Override + public InternalAggregation buildAggregation(long bucket) throws IOException { + profileBreakdown.startTime(AggregationTimingType.BUILD_AGGREGATION); + InternalAggregation result = delegate.buildAggregation(bucket); + profileBreakdown.stopAndRecordTime(); + return result; + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return delegate.buildEmptyAggregation(); + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException { + return new ProfilingLeafBucketCollector(delegate.getLeafCollector(ctx), profileBreakdown); + } + + @Override + public void preCollection() throws IOException { + this.profileBreakdown = profiler.getQueryBreakdown(delegate); + profileBreakdown.startTime(AggregationTimingType.INITIALIZE); + delegate.preCollection(); + profileBreakdown.stopAndRecordTime(); + profiler.pollLastElement(); + } + + @Override + public void postCollection() throws IOException { + delegate.postCollection(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java new file mode 100644 index 00000000000..75c90ded709 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.elasticsearch.search.aggregations.LeafBucketCollector; + +import java.io.IOException; + +public class ProfilingLeafBucketCollector extends LeafBucketCollector { + + private LeafBucketCollector delegate; + private AggregationProfileBreakdown profileBreakdown; + + public ProfilingLeafBucketCollector(LeafBucketCollector delegate, AggregationProfileBreakdown profileBreakdown) { + this.delegate = delegate; + this.profileBreakdown = profileBreakdown; + } + + @Override + public void collect(int doc, long bucket) throws IOException { + profileBreakdown.startTime(AggregationTimingType.COLLECT); + delegate.collect(doc, bucket); + profileBreakdown.stopAndRecordTime(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java index 5b92ef8b2a9..013b7d3a506 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java @@ -20,89 +20,33 @@ package org.elasticsearch.search.profile.query; import org.apache.lucene.search.Query; +import org.elasticsearch.search.profile.AbstractInternalProfileTree; import org.elasticsearch.search.profile.ProfileResult; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.List; -import java.util.Map; -import java.util.concurrent.LinkedBlockingDeque; - /** * This class tracks the dependency tree for queries (scoring and rewriting) and * generates {@link QueryProfileBreakdown} for each node in the tree. It also finalizes the tree * and returns a list of {@link ProfileResult} that can be serialized back to the client */ -final class InternalQueryProfileTree { - - private ArrayList timings; - - /** Maps the Query to it's list of children. This is basically the dependency tree */ - private ArrayList> tree; - - /** A list of the original queries, keyed by index position */ - private ArrayList queries; - - /** A list of top-level "roots". Each root can have its own tree of profiles */ - private ArrayList roots; +final class InternalQueryProfileTree extends AbstractInternalProfileTree { /** Rewrite time */ private long rewriteTime; private long rewriteScratch; - /** A temporary stack used to record where we are in the dependency tree. Only used by scoring queries */ - private Deque stack; - - private int currentToken = 0; - - public InternalQueryProfileTree() { - timings = new ArrayList<>(10); - stack = new LinkedBlockingDeque<>(10); - tree = new ArrayList<>(10); - queries = new ArrayList<>(10); - roots = new ArrayList<>(10); + @Override + protected QueryProfileBreakdown createProfileBreakdown() { + return new QueryProfileBreakdown(); } - /** - * Returns a {@link QueryProfileBreakdown} for a scoring query. Scoring queries (e.g. those - * that are past the rewrite phase and are now being wrapped by createWeight() ) follow - * a recursive progression. We can track the dependency tree by a simple stack - * - * The only hiccup is that the first scoring query will be identical to the last rewritten - * query, so we need to take special care to fix that - * - * @param query The scoring query we wish to profile - * @return A ProfileBreakdown for this query - */ - public QueryProfileBreakdown getQueryBreakdown(Query query) { - int token = currentToken; + @Override + protected String getTypeFromElement(Query query) { + return query.getClass().getSimpleName(); + } - boolean stackEmpty = stack.isEmpty(); - - // If the stack is empty, we are a new root query - if (stackEmpty) { - - // We couldn't find a rewritten query to attach to, so just add it as a - // top-level root. This is just a precaution: it really shouldn't happen. - // We would only get here if a top-level query that never rewrites for some reason. - roots.add(token); - - // Increment the token since we are adding a new node, but notably, do not - // updateParent() because this was added as a root - currentToken += 1; - stack.add(token); - - return addDependencyNode(query, token); - } - - updateParent(token); - - // Increment the token since we are adding a new node - currentToken += 1; - stack.add(token); - - return addDependencyNode(query, token); + @Override + protected String getDescriptionFromElement(Query query) { + return query.toString(); } /** @@ -128,113 +72,7 @@ final class InternalQueryProfileTree { return time; } - /** - * Helper method to add a new node to the dependency tree. - * - * Initializes a new list in the dependency tree, saves the query and - * generates a new {@link QueryProfileBreakdown} to track the timings - * of this query - * - * @param query The query to profile - * @param token The assigned token for this query - * @return A ProfileBreakdown to profile this query - */ - private QueryProfileBreakdown addDependencyNode(Query query, int token) { - - // Add a new slot in the dependency tree - tree.add(new ArrayList<>(5)); - - // Save our query for lookup later - queries.add(query); - - QueryProfileBreakdown queryTimings = new QueryProfileBreakdown(); - timings.add(token, queryTimings); - return queryTimings; - } - - /** - * Removes the last (e.g. most recent) value on the stack - */ - public void pollLast() { - stack.pollLast(); - } - - /** - * After the query has been run and profiled, we need to merge the flat timing map - * with the dependency graph to build a data structure that mirrors the original - * query tree - * - * @return a hierarchical representation of the profiled query tree - */ - public List getQueryTree() { - ArrayList results = new ArrayList<>(5); - for (Integer root : roots) { - results.add(doGetQueryTree(root)); - } - return results; - } - - /** - * Recursive helper to finalize a node in the dependency tree - * @param token The node we are currently finalizing - * @return A hierarchical representation of the tree inclusive of children at this level - */ - private ProfileResult doGetQueryTree(int token) { - Query query = queries.get(token); - QueryProfileBreakdown breakdown = timings.get(token); - Map timings = breakdown.toTimingMap(); - List children = tree.get(token); - List childrenProfileResults = Collections.emptyList(); - - if (children != null) { - childrenProfileResults = new ArrayList<>(children.size()); - for (Integer child : children) { - ProfileResult childNode = doGetQueryTree(child); - childrenProfileResults.add(childNode); - } - } - - // TODO this would be better done bottom-up instead of top-down to avoid - // calculating the same times over and over...but worth the effort? - long nodeTime = getNodeTime(timings, childrenProfileResults); - String queryDescription = query.getClass().getSimpleName(); - String luceneName = query.toString(); - return new ProfileResult(queryDescription, luceneName, timings, childrenProfileResults, nodeTime); - } - public long getRewriteTime() { return rewriteTime; } - - /** - * Internal helper to add a child to the current parent node - * - * @param childToken The child to add to the current parent - */ - private void updateParent(int childToken) { - Integer parent = stack.peekLast(); - ArrayList parentNode = tree.get(parent); - parentNode.add(childToken); - tree.set(parent, parentNode); - } - - /** - * Internal helper to calculate the time of a node, inclusive of children - * - * @param timings A map of breakdown timing for the node - * @param children All children profile results at this node - * @return The total time at this node, inclusive of children - */ - private static long getNodeTime(Map timings, List children) { - long nodeTime = 0; - for (long time : timings.values()) { - nodeTime += time; - } - - // Then add up our children - for (ProfileResult child : children) { - nodeTime += getNodeTime(child.getTimeBreakdown(), child.getProfiledChildren()); - } - return nodeTime; - } } diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java new file mode 100644 index 00000000000..d5e00aca336 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.query; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.profile.ProfileResult; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A container class to hold the profile results for a single shard in the request. + * Contains a list of query profiles, a collector tree and a total rewrite tree. + */ +public final class QueryProfileShardResult implements Writeable, ToXContent { + + private final List queryProfileResults; + + private final CollectorResult profileCollector; + + private final long rewriteTime; + + public QueryProfileShardResult(List queryProfileResults, long rewriteTime, + CollectorResult profileCollector) { + assert(profileCollector != null); + this.queryProfileResults = queryProfileResults; + this.profileCollector = profileCollector; + this.rewriteTime = rewriteTime; + } + + /** + * Read from a stream. + */ + public QueryProfileShardResult(StreamInput in) throws IOException { + int profileSize = in.readVInt(); + queryProfileResults = new ArrayList<>(profileSize); + for (int j = 0; j < profileSize; j++) { + queryProfileResults.add(new ProfileResult(in)); + } + + profileCollector = new CollectorResult(in); + rewriteTime = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(queryProfileResults.size()); + for (ProfileResult p : queryProfileResults) { + p.writeTo(out); + } + profileCollector.writeTo(out); + out.writeLong(rewriteTime); + } + + + public List getQueryResults() { + return Collections.unmodifiableList(queryProfileResults); + } + + public long getRewriteTime() { + return rewriteTime; + } + + public CollectorResult getCollectorResult() { + return profileCollector; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("query"); + for (ProfileResult p : queryProfileResults) { + p.toXContent(builder, params); + } + builder.endArray(); + builder.field("rewrite_time", rewriteTime); + builder.startArray("collector"); + profileCollector.toXContent(builder, params); + builder.endArray(); + return builder; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java index 57341ee132f..0051356e35a 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java @@ -20,9 +20,8 @@ package org.elasticsearch.search.profile.query; import org.apache.lucene.search.Query; -import org.elasticsearch.search.profile.ProfileResult; +import org.elasticsearch.search.profile.AbstractProfiler; -import java.util.List; import java.util.Objects; /** @@ -36,16 +35,16 @@ import java.util.Objects; * request may execute two searches (query + global agg). A Profiler just * represents one of those */ -public final class QueryProfiler { - - private final InternalQueryProfileTree queryTree = new InternalQueryProfileTree(); +public final class QueryProfiler extends AbstractProfiler { /** * The root Collector used in the search */ private InternalProfileCollector collector; - public QueryProfiler() {} + public QueryProfiler() { + super(new InternalQueryProfileTree()); + } /** Set the collector that is associated with this profiler. */ public void setCollector(InternalProfileCollector collector) { @@ -55,21 +54,12 @@ public final class QueryProfiler { this.collector = Objects.requireNonNull(collector); } - /** - * Get the {@link QueryProfileBreakdown} for the given query, potentially creating it if it did not exist. - * This should only be used for queries that will be undergoing scoring. Do not use it to profile the - * rewriting phase - */ - public QueryProfileBreakdown getQueryBreakdown(Query query) { - return queryTree.getQueryBreakdown(query); - } - /** * Begin timing the rewrite phase of a request. All rewrites are accumulated together into a * single metric */ public void startRewriteTime() { - queryTree.startRewriteTime(); + ((InternalQueryProfileTree) profileTree).startRewriteTime(); } /** @@ -79,29 +69,14 @@ public final class QueryProfiler { * @return cumulative rewrite time */ public long stopAndAddRewriteTime() { - return queryTree.stopAndAddRewriteTime(); - } - - /** - * Removes the last (e.g. most recent) query on the stack. This should only be called for scoring - * queries, not rewritten queries - */ - public void pollLastQuery() { - queryTree.pollLast(); - } - - /** - * @return a hierarchical representation of the profiled query tree - */ - public List getQueryTree() { - return queryTree.getQueryTree(); + return ((InternalQueryProfileTree) profileTree).stopAndAddRewriteTime(); } /** * @return total time taken to rewrite all queries in this profile */ public long getRewriteTime() { - return queryTree.getRewriteTime(); + return ((InternalQueryProfileTree) profileTree).getRewriteTime(); } /** diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index df68064f617..4e3a642f694 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -42,10 +42,10 @@ import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.Weight; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.MinimumScoreCollector; import org.elasticsearch.common.lucene.search.FilteredCollector; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; @@ -76,11 +76,10 @@ public class QueryPhase implements SearchPhase { private final SuggestPhase suggestPhase; private RescorePhase rescorePhase; - @Inject - public QueryPhase(AggregationPhase aggregationPhase, SuggestPhase suggestPhase, RescorePhase rescorePhase) { - this.aggregationPhase = aggregationPhase; - this.suggestPhase = suggestPhase; - this.rescorePhase = rescorePhase; + public QueryPhase(Settings settings) { + this.aggregationPhase = new AggregationPhase(); + this.suggestPhase = new SuggestPhase(settings); + this.rescorePhase = new RescorePhase(settings); } @Override @@ -112,8 +111,8 @@ public class QueryPhase implements SearchPhase { aggregationPhase.execute(searchContext); if (searchContext.getProfilers() != null) { - List shardResults = SearchProfileShardResults - .buildShardResults(searchContext.getProfilers().getQueryProfilers()); + ProfileShardResult shardResults = SearchProfileShardResults + .buildShardResults(searchContext.getProfilers()); searchContext.queryResult().profileResults(shardResults); } } @@ -385,8 +384,8 @@ public class QueryPhase implements SearchPhase { queryResult.topDocs(topDocsCallable.call(), sortValueFormats); if (searchContext.getProfilers() != null) { - List shardResults = SearchProfileShardResults - .buildShardResults(searchContext.getProfilers().getQueryProfilers()); + ProfileShardResult shardResults = SearchProfileShardResults + .buildShardResults(searchContext.getProfilers()); searchContext.queryResult().profileResults(shardResults); } diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 1408ebe8359..be8c895eecd 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -59,7 +59,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { private Suggest suggest; private boolean searchTimedOut; private Boolean terminatedEarly = null; - private List profileShardResults; + private ProfileShardResult profileShardResults; public QuerySearchResult() { @@ -143,7 +143,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { * Returns the profiled results for this search, or potentially null if result was empty * @return The profiled results, or null */ - public @Nullable List profileResults() { + public @Nullable ProfileShardResult profileResults() { return profileShardResults; } @@ -151,7 +151,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { * Sets the finalized profiling results for this query * @param shardResults The finalized profile */ - public void profileResults(List shardResults) { + public void profileResults(ProfileShardResult shardResults) { this.profileShardResults = shardResults; } @@ -237,12 +237,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { terminatedEarly = in.readOptionalBoolean(); if (in.getVersion().onOrAfter(Version.V_2_2_0) && in.readBoolean()) { - int profileSize = in.readVInt(); - profileShardResults = new ArrayList<>(profileSize); - for (int i = 0; i < profileSize; i++) { - ProfileShardResult result = new ProfileShardResult(in); - profileShardResults.add(result); - } + profileShardResults = new ProfileShardResult(in); } } @@ -296,10 +291,7 @@ public class QuerySearchResult extends QuerySearchResultProvider { out.writeBoolean(false); } else { out.writeBoolean(true); - out.writeVInt(profileShardResults.size()); - for (ProfileShardResult shardResult : profileShardResults) { - shardResult.writeTo(out); - } + profileShardResults.writeTo(out); } } } diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java index b82ed941e1c..395db4cdcd8 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java @@ -22,34 +22,20 @@ package org.elasticsearch.search.rescore; import org.apache.lucene.search.TopDocs; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; -import java.util.Collections; -import java.util.Map; /** */ public class RescorePhase extends AbstractComponent implements SearchPhase { - @Inject public RescorePhase(Settings settings) { super(settings); } - /** - * rescorers do not have a parse element, they use - * {@link RescoreBuilder#parseFromXContent(org.elasticsearch.index.query.QueryParseContext)} for parsing instead. - */ - @Override - public Map parseElements() { - return Collections.emptyMap(); - } - @Override public void preProcess(SearchContext context) { } diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java index 0d87b275403..2b8040ebd28 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java @@ -61,7 +61,7 @@ public abstract class SliceQuery extends Query { @Override public boolean equals(Object o) { - if (super.equals(o) == false) { + if (sameClassAs(o) == false) { return false; } SliceQuery that = (SliceQuery) o; @@ -70,7 +70,7 @@ public abstract class SliceQuery extends Query { @Override public int hashCode() { - return Objects.hash(super.hashCode(), field, id, max); + return Objects.hash(classHash(), field, id, max); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java index b967a6b6e71..429a3ebe892 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java @@ -74,11 +74,7 @@ public final class TermsSliceQuery extends SliceQuery { int hashCode = term.hashCode(); if (contains(hashCode)) { docsEnum = te.postings(docsEnum, PostingsEnum.NONE); - int docId = docsEnum.nextDoc(); - while (docId != DocIdSetIterator.NO_MORE_DOCS) { - builder.add(docId); - docId = docsEnum.nextDoc(); - } + builder.add(docsEnum); } } return builder.build(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java index 520322587bb..c0567e59e8c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java @@ -21,9 +21,7 @@ package org.elasticsearch.search.suggest; import org.apache.lucene.util.CharsRefBuilder; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.suggest.Suggest.Suggestion; @@ -36,24 +34,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import static java.util.Collections.emptyMap; - /** */ public class SuggestPhase extends AbstractComponent implements SearchPhase { - @Inject public SuggestPhase(Settings settings) { super(settings); } - @Override - public Map parseElements() { - // this is used to parse SearchSourceBuilder.ext() bytes - // we don't allow any suggestion parsing for the extension - return emptyMap(); - } - @Override public void preProcess(SearchContext context) { } diff --git a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java new file mode 100644 index 00000000000..7d551ce195c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java @@ -0,0 +1,224 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.tasks; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap; + +/** + * Information about a persisted or running task. Running tasks just have a {@link #getTask()} while persisted tasks will have either a + * {@link #getError()} or {@link #getResult()}. + */ +public final class PersistedTaskInfo implements Writeable, ToXContent { + private final TaskInfo task; + @Nullable + private final BytesReference error; + @Nullable + private final BytesReference result; + + /** + * Construct a {@linkplain PersistedTaskInfo} for a running task. + */ + public PersistedTaskInfo(TaskInfo task) { + this(task, null, null); + } + + /** + * Construct a {@linkplain PersistedTaskInfo} for a task that completed with an error. + */ + public PersistedTaskInfo(TaskInfo task, Throwable error) throws IOException { + this(task, toXContent(error), null); + } + + /** + * Construct a {@linkplain PersistedTaskInfo} for a task that completed successfully. + */ + public PersistedTaskInfo(TaskInfo task, ToXContent result) throws IOException { + this(task, null, toXContent(result)); + } + + private PersistedTaskInfo(TaskInfo task, @Nullable BytesReference error, @Nullable BytesReference result) { + this.task = requireNonNull(task, "task is required"); + this.error = error; + this.result = result; + } + + /** + * Read from a stream. + */ + public PersistedTaskInfo(StreamInput in) throws IOException { + task = new TaskInfo(in); + error = in.readOptionalBytesReference(); + result = in.readOptionalBytesReference(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + task.writeTo(out); + out.writeOptionalBytesReference(error); + out.writeOptionalBytesReference(result); + } + + /** + * Get the task that this wraps. + */ + public TaskInfo getTask() { + return task; + } + + /** + * Get the error that finished this task. Will return null if the task didn't finish with an error or it hasn't yet finished. + */ + public BytesReference getError() { + return error; + } + + /** + * Convert {@link #getError()} from XContent to a Map for easy processing. Will return null if the task didn't finish with an error or + * hasn't yet finished. + */ + public Map getErrorAsMap() { + if (error == null) { + return null; + } + return convertToMap(error, false).v2(); + } + + /** + * Get the result that this task finished with. Will return null if the task was finished by an error or it hasn't yet finished. + */ + public BytesReference getResult() { + return result; + } + + /** + * Convert {@link #getResult()} from XContent to a Map for easy processing. Will return null if the task was finished with an error or + * hasn't yet finished. + */ + public Map getResultAsMap() { + if (result == null) { + return null; + } + return convertToMap(result, false).v2(); + } + + /** + * Was the task completed before returned? + */ + public boolean isCompleted() { + return error != null || result != null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + innerToXContent(builder, params); + return builder.endObject(); + } + + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("task", task); + if (error != null) { + XContentHelper.writeRawField("error", error, builder, params); + } + if (result != null) { + XContentHelper.writeRawField("result", result, builder, params); + } + return builder; + } + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "persisted_task_info", a -> new PersistedTaskInfo((TaskInfo) a[0], (BytesReference) a[1], (BytesReference) a[2])); + static { + PARSER.declareObject(constructorArg(), TaskInfo.PARSER, new ParseField("task")); + PARSER.declareRawObject(optionalConstructorArg(), new ParseField("error")); + PARSER.declareRawObject(optionalConstructorArg(), new ParseField("result")); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + // Implements equals and hashcode for testing + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != PersistedTaskInfo.class) { + return false; + } + PersistedTaskInfo other = (PersistedTaskInfo) obj; + /* + * Equality of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing + * differences so perfect for testing. + */ + return Objects.equals(task, other.task) + && Objects.equals(getErrorAsMap(), other.getErrorAsMap()) + && Objects.equals(getResultAsMap(), other.getResultAsMap()); + } + + @Override + public int hashCode() { + /* + * Hashing of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing + * differences so perfect for testing. + */ + return Objects.hash(task, getErrorAsMap(), getResultAsMap()); + } + + private static BytesReference toXContent(ToXContent result) throws IOException { + try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + // Elasticsearch's Response object never emit starting or ending objects. Most other implementers of ToXContent do.... + builder.startObject(); + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + return builder.bytes(); + } + } + + private static BytesReference toXContent(Throwable error) throws IOException { + try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + builder.startObject(); + ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, error); + builder.endObject(); + return builder.bytes(); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java b/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java new file mode 100644 index 00000000000..a540dbd8e3c --- /dev/null +++ b/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tasks; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +import static java.util.Objects.requireNonNull; +import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap; + +/** + * Raw, unparsed status from the task results index. + */ +public class RawTaskStatus implements Task.Status { + public static final String NAME = "raw"; + + private final BytesReference status; + + public RawTaskStatus(BytesReference status) { + this.status = requireNonNull(status, "status may not be null"); + } + + /** + * Read from a stream. + */ + public RawTaskStatus(StreamInput in) throws IOException { + status = in.readOptionalBytesReference(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalBytesReference(status); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.rawValue(status); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + /** + * Convert the from XContent to a Map for easy reading. + */ + public Map toMap() { + return convertToMap(status, false).v2(); + } + + // Implements equals and hashcode for testing + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != RawTaskStatus.class) { + return false; + } + RawTaskStatus other = (RawTaskStatus) obj; + // Totally not efficient, but ok for testing because it ignores order and spacing differences + return toMap().equals(other.toMap()); + } + + @Override + public int hashCode() { + // Totally not efficient, but ok for testing because consistent with equals + return toMap().hashCode(); + } +} diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java index 9c3f4cc842d..8c0b599e933 100644 --- a/core/src/main/java/org/elasticsearch/tasks/Task.java +++ b/core/src/main/java/org/elasticsearch/tasks/Task.java @@ -21,7 +21,6 @@ package org.elasticsearch.tasks; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; @@ -78,8 +77,8 @@ public class Task { description = getDescription(); status = getStatus(); } - return new TaskInfo(node, getId(), getType(), getAction(), description, status, startTime, System.nanoTime() - startTimeNanos, - this instanceof CancellableTask, parentTask); + return new TaskInfo(new TaskId(node.getId(), getId()), getType(), getAction(), description, status, startTime, + System.nanoTime() - startTimeNanos, this instanceof CancellableTask, parentTask); } /** @@ -136,13 +135,13 @@ public class Task { public interface Status extends ToXContent, NamedWriteable {} - public TaskResult result(DiscoveryNode node, Throwable error) throws IOException { - return new TaskResult(taskInfo(node, true), error); + public PersistedTaskInfo result(DiscoveryNode node, Throwable error) throws IOException { + return new PersistedTaskInfo(taskInfo(node, true), error); } - public TaskResult result(DiscoveryNode node, ActionResponse response) throws IOException { + public PersistedTaskInfo result(DiscoveryNode node, ActionResponse response) throws IOException { if (response instanceof ToXContent) { - return new TaskResult(taskInfo(node, true), (ToXContent) response); + return new PersistedTaskInfo(taskInfo(node, true), (ToXContent) response); } else { throw new IllegalStateException("response has to implement ToXContent for persistence"); } diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java similarity index 54% rename from core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java rename to core/src/main/java/org/elasticsearch/tasks/TaskInfo.java index adc3afb1bcd..1bd5f1bfc6a 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java @@ -17,20 +17,27 @@ * under the License. */ -package org.elasticsearch.action.admin.cluster.node.tasks.list; +package org.elasticsearch.tasks; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + + /** * Information about a currently running task. *

@@ -39,10 +46,7 @@ import java.util.concurrent.TimeUnit; * and use in APIs. Instead, immutable and streamable TaskInfo objects are used to represent * snapshot information about currently running tasks. */ -public class TaskInfo implements Writeable, ToXContent { - - private final DiscoveryNode node; - +public final class TaskInfo implements Writeable, ToXContent { private final TaskId taskId; private final String type; @@ -61,10 +65,9 @@ public class TaskInfo implements Writeable, ToXContent { private final TaskId parentTaskId; - public TaskInfo(DiscoveryNode node, long id, String type, String action, String description, Task.Status status, long startTime, + public TaskInfo(TaskId taskId, String type, String action, String description, Task.Status status, long startTime, long runningTimeNanos, boolean cancellable, TaskId parentTaskId) { - this.node = node; - this.taskId = new TaskId(node.getId(), id); + this.taskId = taskId; this.type = type; this.action = action; this.description = description; @@ -79,8 +82,7 @@ public class TaskInfo implements Writeable, ToXContent { * Read from a stream. */ public TaskInfo(StreamInput in) throws IOException { - node = new DiscoveryNode(in); - taskId = new TaskId(node.getId(), in.readLong()); + taskId = TaskId.readFromStream(in); type = in.readString(); action = in.readString(); description = in.readOptionalString(); @@ -93,8 +95,7 @@ public class TaskInfo implements Writeable, ToXContent { @Override public void writeTo(StreamOutput out) throws IOException { - node.writeTo(out); - out.writeLong(taskId.getId()); + taskId.writeTo(out); out.writeString(type); out.writeString(action); out.writeOptionalString(description); @@ -109,10 +110,6 @@ public class TaskInfo implements Writeable, ToXContent { return taskId; } - public DiscoveryNode getNode() { - return node; - } - public long getId() { return taskId.getId(); } @@ -167,7 +164,13 @@ public class TaskInfo implements Writeable, ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("node", node.getId()); + builder.startObject(); + innerToXContent(builder, params); + return builder.endObject(); + } + + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("node", taskId.getNodeId()); builder.field("id", taskId.getId()); builder.field("type", type); builder.field("action", action); @@ -185,4 +188,63 @@ public class TaskInfo implements Writeable, ToXContent { } return builder; } + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "task_info", a -> { + int i = 0; + TaskId id = new TaskId((String) a[i++], (Long) a[i++]); + String type = (String) a[i++]; + String action = (String) a[i++]; + String description = (String) a[i++]; + BytesReference statusBytes = (BytesReference) a[i++]; + long startTime = (Long) a[i++]; + long runningTimeNanos = (Long) a[i++]; + boolean cancellable = (Boolean) a[i++]; + String parentTaskIdString = (String) a[i++]; + + RawTaskStatus status = statusBytes == null ? null : new RawTaskStatus(statusBytes); + TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId((String) parentTaskIdString); + return new TaskInfo(id, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId); + }); + static { + // Note for the future: this has to be backwards compatible with all changes to the task persistence format + PARSER.declareString(constructorArg(), new ParseField("node")); + PARSER.declareLong(constructorArg(), new ParseField("id")); + PARSER.declareString(constructorArg(), new ParseField("type")); + PARSER.declareString(constructorArg(), new ParseField("action")); + PARSER.declareString(optionalConstructorArg(), new ParseField("description")); + PARSER.declareRawObject(optionalConstructorArg(), new ParseField("status")); + PARSER.declareLong(constructorArg(), new ParseField("start_time_in_millis")); + PARSER.declareLong(constructorArg(), new ParseField("running_time_in_nanos")); + PARSER.declareBoolean(constructorArg(), new ParseField("cancellable")); + PARSER.declareString(optionalConstructorArg(), new ParseField("parent_task_id")); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + // Implements equals and hashCode for testing + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != TaskInfo.class) { + return false; + } + TaskInfo other = (TaskInfo) obj; + return Objects.equals(taskId, other.taskId) + && Objects.equals(type, other.type) + && Objects.equals(action, other.action) + && Objects.equals(description, other.description) + && Objects.equals(startTime, other.startTime) + && Objects.equals(runningTimeNanos, other.runningTimeNanos) + && Objects.equals(parentTaskId, other.parentTaskId) + && Objects.equals(cancellable, other.cancellable) + && Objects.equals(status, other.status); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, type, action, description, startTime, runningTimeNanos, parentTaskId, cancellable, status); + } } diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java index c44eb65301a..045522f3288 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java @@ -57,7 +57,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen private final Map banedParents = new ConcurrentHashMap<>(); - private TaskResultsService taskResultsService; + private TaskPersistenceService taskResultsService; private DiscoveryNodes lastDiscoveryNodes = DiscoveryNodes.EMPTY_NODES; @@ -65,7 +65,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen super(settings); } - public void setTaskResultsService(TaskResultsService taskResultsService) { + public void setTaskResultsService(TaskPersistenceService taskResultsService) { assert this.taskResultsService == null; this.taskResultsService = taskResultsService; } @@ -145,14 +145,14 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen /** * Stores the task failure */ - public void persistResult(Task task, Throwable error, ActionListener listener) { + public void persistResult(Task task, Throwable error, ActionListener listener) { DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode(); if (localNode == null) { // too early to persist anything, shouldn't really be here - just pass the error along listener.onFailure(error); return; } - final TaskResult taskResult; + final PersistedTaskInfo taskResult; try { taskResult = task.result(localNode, error); } catch (IOException ex) { @@ -177,7 +177,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen /** * Stores the task result */ - public void persistResult(Task task, Response response, ActionListener listener) { + public void persistResult(Task task, Response response, ActionListener listener) { DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode(); if (localNode == null) { // too early to persist anything, shouldn't really be here - just pass the response along @@ -185,7 +185,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen listener.onResponse(response); return; } - final TaskResult taskResult; + final PersistedTaskInfo taskResult; try { taskResult = task.result(localNode, response); } catch (IOException ex) { diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java similarity index 71% rename from core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java rename to core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java index c0f63868d3d..e1a26663e7b 100644 --- a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java @@ -19,14 +19,17 @@ package org.elasticsearch.tasks; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.service.ClusterService; @@ -34,21 +37,25 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.indices.IndexAlreadyExistsException; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; /** - * Service that can persist task results + * Service that can persist tasks and their results. */ -public class TaskResultsService extends AbstractComponent { +public class TaskPersistenceService extends AbstractComponent { - public static final String TASK_RESULT_INDEX = ".results"; + public static final String TASK_INDEX = ".tasks"; - public static final String TASK_RESULT_TYPE = "result"; + public static final String TASK_TYPE = "task"; - public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-results-index-mapping.json"; + public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-index-mapping.json"; private final Client client; @@ -57,7 +64,7 @@ public class TaskResultsService extends AbstractComponent { private final TransportCreateIndexAction createIndexAction; @Inject - public TaskResultsService(Settings settings, Client client, ClusterService clusterService, + public TaskPersistenceService(Settings settings, Client client, ClusterService clusterService, TransportCreateIndexAction createIndexAction) { super(settings); this.client = client; @@ -65,15 +72,15 @@ public class TaskResultsService extends AbstractComponent { this.createIndexAction = createIndexAction; } - public void persist(TaskResult taskResult, ActionListener listener) { + public void persist(PersistedTaskInfo taskResult, ActionListener listener) { ClusterState state = clusterService.state(); - if (state.routingTable().hasIndex(TASK_RESULT_INDEX) == false) { + if (state.routingTable().hasIndex(TASK_INDEX) == false) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.settings(taskResultIndexSettings()); - createIndexRequest.index(TASK_RESULT_INDEX); - createIndexRequest.mapping(TASK_RESULT_TYPE, taskResultIndexMapping()); + createIndexRequest.index(TASK_INDEX); + createIndexRequest.mapping(TASK_TYPE, taskResultIndexMapping()); createIndexRequest.cause("auto(task api)"); createIndexAction.execute(null, createIndexRequest, new ActionListener() { @@ -97,10 +104,10 @@ public class TaskResultsService extends AbstractComponent { } }); } else { - IndexMetaData metaData = state.getMetaData().index(TASK_RESULT_INDEX); - if (metaData.getMappings().containsKey(TASK_RESULT_TYPE) == false) { + IndexMetaData metaData = state.getMetaData().index(TASK_INDEX); + if (metaData.getMappings().containsKey(TASK_TYPE) == false) { // The index already exists but doesn't have our mapping - client.admin().indices().preparePutMapping(TASK_RESULT_INDEX).setType(TASK_RESULT_TYPE).setSource(taskResultIndexMapping()) + client.admin().indices().preparePutMapping(TASK_INDEX).setType(TASK_TYPE).setSource(taskResultIndexMapping()) .execute(new ActionListener() { @Override public void onResponse(PutMappingResponse putMappingResponse) { @@ -120,20 +127,25 @@ public class TaskResultsService extends AbstractComponent { } - private void doPersist(TaskResult taskResult, ActionListener listener) { - client.prepareIndex(TASK_RESULT_INDEX, TASK_RESULT_TYPE, taskResult.getTaskId().toString()).setSource(taskResult.getResult()) - .execute(new ActionListener() { - @Override - public void onResponse(IndexResponse indexResponse) { - listener.onResponse(null); - } - - @Override - public void onFailure(Throwable e) { - listener.onFailure(e); - } - }); + private void doPersist(PersistedTaskInfo taskResult, ActionListener listener) { + IndexRequestBuilder index = client.prepareIndex(TASK_INDEX, TASK_TYPE, taskResult.getTask().getTaskId().toString()); + try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS); + index.setSource(builder); + } catch (IOException e) { + throw new ElasticsearchException("Couldn't convert task result to XContent for [{}]", e, taskResult.getTask()); + } + index.execute(new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + listener.onResponse(null); + } + @Override + public void onFailure(Throwable e) { + listener.onFailure(e); + } + }); } private Settings taskResultIndexSettings() { diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java deleted file mode 100644 index a33a7842c27..00000000000 --- a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.tasks; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; -import org.elasticsearch.client.Requests; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; - -import java.io.IOException; - -/** - * Represents the result or failure of a running task - */ -public class TaskResult { - - private final BytesReference result; - - private final TaskId taskId; - - public TaskResult(TaskInfo taskInfo, Throwable e) throws IOException { - ToXContent.Params params = ToXContent.EMPTY_PARAMS; - XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE); - builder.startObject(); - { - builder.startObject("task"); - { - taskInfo.toXContent(builder, params); - } - builder.endObject(); - builder.startObject("error"); - { - ElasticsearchException.toXContent(builder, params, e); - } - builder.endObject(); - } - builder.endObject(); - result = builder.bytes(); - taskId = taskInfo.getTaskId(); - } - - public TaskResult(TaskInfo taskInfo, ToXContent toXContent) throws IOException { - ToXContent.Params params = ToXContent.EMPTY_PARAMS; - XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE); - builder.startObject(); - { - builder.startObject("task"); - { - taskInfo.toXContent(builder, params); - } - builder.endObject(); - builder.startObject("result"); - { - toXContent.toXContent(builder, params); - } - builder.endObject(); - } - builder.endObject(); - result = builder.bytes(); - taskId = taskInfo.getTaskId(); - } - - public TaskId getTaskId() { - return taskId; - } - - public BytesReference getResult() { - return result; - } -} diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 5c31323b3d8..3f6a65c82c5 100644 --- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -167,7 +167,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200)); builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 50)); builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, availableProcessors, 1000)); - builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, ((availableProcessors * 3) / 2) + 1, 1000)); + builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(availableProcessors), 1000)); builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5))); // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded // the assumption here is that the listeners should be very lightweight on the listeners side @@ -389,6 +389,10 @@ public class ThreadPool extends AbstractComponent implements Closeable { return boundedBy(2 * numberOfProcessors, 2, Integer.MAX_VALUE); } + public static int searchThreadPoolSize(int availableProcessors) { + return ((availableProcessors * 3) / 2) + 1; + } + class LoggingRunnable implements Runnable { private final Runnable runnable; @@ -579,7 +583,7 @@ public class ThreadPool extends AbstractComponent implements Closeable { min = in.readInt(); max = in.readInt(); if (in.readBoolean()) { - keepAlive = TimeValue.readTimeValue(in); + keepAlive = new TimeValue(in); } if (in.readBoolean()) { queueSize = SizeValue.readSizeValue(in); diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java index 91b6bc120ad..03856017c36 100644 --- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java +++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java @@ -42,9 +42,6 @@ import org.jboss.netty.channel.ChannelFutureListener; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; -/** - * - */ public class NettyTransportChannel implements TransportChannel { private final NettyTransport transport; @@ -55,7 +52,7 @@ public class NettyTransportChannel implements TransportChannel { private final long requestId; private final String profileName; private final long reservedBytes; - private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean released = new AtomicBoolean(); public NettyTransportChannel(NettyTransport transport, TransportServiceAdapter transportServiceAdapter, String action, Channel channel, long requestId, Version version, String profileName, long reservedBytes) { @@ -86,7 +83,7 @@ public class NettyTransportChannel implements TransportChannel { @Override public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { - close(); + release(); if (transport.compress) { options = TransportResponseOptions.builder(options).withCompress(transport.compress).build(); } @@ -128,7 +125,7 @@ public class NettyTransportChannel implements TransportChannel { @Override public void sendResponse(Throwable error) throws IOException { - close(); + release(); BytesStreamOutput stream = new BytesStreamOutput(); stream.skip(NettyHeader.HEADER_SIZE); RemoteTransportException tx = new RemoteTransportException( @@ -147,10 +144,10 @@ public class NettyTransportChannel implements TransportChannel { future.addListener(onResponseSentListener); } - private void close() { - // attempt to close once atomically - if (closed.compareAndSet(false, true) == false) { - throw new IllegalStateException("Channel is already closed"); + private void release() { + // attempt to release once atomically + if (released.compareAndSet(false, true) == false) { + throw new IllegalStateException("reserved bytes are already released"); } transport.inFlightRequestsBreaker().addWithoutBreaking(-reservedBytes); } @@ -174,4 +171,5 @@ public class NettyTransportChannel implements TransportChannel { public Channel getChannel() { return channel; } + } diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index ff0ea773667..b185289b58d 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-6.0.1.jar}" { +grant codeBase "${codebase.lucene-core-6.1.0-snapshot-3a57bea.jar}" { // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die) // java 8 package permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; @@ -42,6 +42,11 @@ grant codeBase "${codebase.lucene-core-6.0.1.jar}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; +grant codeBase "${codebase.lucene-misc-6.1.0-snapshot-3a57bea.jar}" { + // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper + permission java.nio.file.LinkPermission "hard"; +}; + //// Everything else: grant { diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index d4ab6e01ab9..7abebf01be1 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -24,6 +24,8 @@ grant codeBase "${codebase.securemock-1.2.jar}" { // needed to access ReflectionFactory (see below) permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect"; + // needed for reflection in ibm jdk + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; // needed to support creation of mocks permission java.lang.RuntimePermission "reflectionFactoryAccess"; // needed for spy interception, etc @@ -31,9 +33,11 @@ grant codeBase "${codebase.securemock-1.2.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-6.0.1.jar}" { +grant codeBase "${codebase.lucene-test-framework-6.1.0-snapshot-3a57bea.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + // needed for testing hardlinks in StoreRecoveryTests since we install MockFS + permission java.nio.file.LinkPermission "hard"; }; grant codeBase "${codebase.randomizedtesting-runner-2.3.2.jar}" { diff --git a/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json similarity index 98% rename from core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json rename to core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json index 0310418bb43..30476b1fe3a 100644 --- a/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json +++ b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json @@ -1,5 +1,5 @@ { - "result" : { + "task" : { "dynamic" : "strict", "properties" : { "task" : { diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java index 65c91f5daab..862cccab318 100644 --- a/core/src/test/java/org/elasticsearch/VersionTests.java +++ b/core/src/test/java/org/elasticsearch/VersionTests.java @@ -270,7 +270,8 @@ public class VersionTests extends ESTestCase { assertTrue("lucene versions must be " + other + " >= " + version, other.luceneVersion.onOrAfter(version.luceneVersion)); } - if (other.major == version.major && other.minor == version.minor) { + if (other.isAlpha() == false && version.isAlpha() == false + && other.major == version.major && other.minor == version.minor) { assertEquals(other.luceneVersion.major, version.luceneVersion.major); assertEquals(other.luceneVersion.minor, version.luceneVersion.minor); // should we also assert the lucene bugfix version? diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java index c35a9e91314..b5e113dfcd3 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksReque import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.action.support.nodes.BaseNodeRequest; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; @@ -38,6 +37,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java index 025d2b39f6d..2aad88c99a9 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java @@ -19,11 +19,11 @@ package org.elasticsearch.action.admin.cluster.node.tasks; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.tasks.MockTaskManagerListener; import java.util.ArrayList; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index a83a05b8ad8..95e8cde0a22 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -233,5 +233,4 @@ public abstract class TaskManagerTestCase extends ESTestCase { } return listeners; } - } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 270059b09dc..376bd7f3ffa 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -19,14 +19,17 @@ package org.elasticsearch.action.admin.cluster.node.tasks; import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction; @@ -43,8 +46,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.PersistedTaskInfo; import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskResultsService; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.tasks.TaskPersistenceService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.test.tasks.MockTaskManagerListener; @@ -59,22 +65,26 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.function.Function; +import static java.util.Collections.singleton; import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyCollectionOf; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; @@ -199,7 +209,7 @@ public class TasksIT extends ESIntegTestCase { logger.debug("number of shards, total: [{}], primaries: [{}] ", numberOfShards.totalNumShards, numberOfShards.numPrimaries); logger.debug("main events {}", numberOfEvents(RefreshAction.NAME, Tuple::v1)); - logger.debug("main event node {}", findEvents(RefreshAction.NAME, Tuple::v1).get(0).getNode().getName()); + logger.debug("main event node {}", findEvents(RefreshAction.NAME, Tuple::v1).get(0).getTaskId().getNodeId()); logger.debug("[s] events {}", numberOfEvents(RefreshAction.NAME + "[s]", Tuple::v1)); logger.debug("[s][*] events {}", numberOfEvents(RefreshAction.NAME + "[s][*]", Tuple::v1)); logger.debug("nodes with the index {}", internalCluster().nodesInclude("test")); @@ -219,15 +229,16 @@ public class TasksIT extends ESIntegTestCase { TaskInfo mainTask = findEvents(RefreshAction.NAME, Tuple::v1).get(0); List sTasks = findEvents(RefreshAction.NAME + "[s]", Tuple::v1); for (TaskInfo taskInfo : sTasks) { - if (mainTask.getNode().equals(taskInfo.getNode())) { + if (mainTask.getTaskId().getNodeId().equals(taskInfo.getTaskId().getNodeId())) { // This shard level task runs on the same node as a parent task - it should have the main task as a direct parent assertParentTask(Collections.singletonList(taskInfo), mainTask); } else { String description = taskInfo.getDescription(); // This shard level task runs on another node - it should have a corresponding shard level task on the node where main task // is running - List sTasksOnRequestingNode = findEvents(RefreshAction.NAME + "[s]", event -> event.v1() - && mainTask.getNode().equals(event.v2().getNode()) && description.equals(event.v2().getDescription())); + List sTasksOnRequestingNode = findEvents(RefreshAction.NAME + "[s]", + event -> event.v1() && mainTask.getTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId()) + && description.equals(event.v2().getDescription())); // There should be only one parent task assertEquals(1, sTasksOnRequestingNode.size()); assertParentTask(Collections.singletonList(taskInfo), sTasksOnRequestingNode.get(0)); @@ -243,12 +254,13 @@ public class TasksIT extends ESIntegTestCase { List sTask; if (taskInfo.getAction().endsWith("[s][p]")) { // A [s][p] level task should have a corresponding [s] level task on the same node - sTask = findEvents(RefreshAction.NAME + "[s]", event -> event.v1() && taskInfo.getNode().equals(event.v2().getNode()) - && taskInfo.getDescription().equals(event.v2().getDescription())); + sTask = findEvents(RefreshAction.NAME + "[s]", + event -> event.v1() && taskInfo.getTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId()) + && taskInfo.getDescription().equals(event.v2().getDescription())); } else { // A [s][r] level task should have a corresponding [s] level task on the a different node (where primary is located) sTask = findEvents(RefreshAction.NAME + "[s]", - event -> event.v1() && taskInfo.getParentTaskId().getNodeId().equals(event.v2().getNode().getId()) && taskInfo + event -> event.v1() && taskInfo.getParentTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId()) && taskInfo .getDescription() .equals(event.v2().getDescription())); } @@ -305,7 +317,7 @@ public class TasksIT extends ESIntegTestCase { // they all should have the same shard task as a parent assertEquals(getNumShards("test").numReplicas, numberOfEvents(BulkAction.NAME + "[s][r]", Tuple::v1)); assertParentTask(findEvents(BulkAction.NAME + "[s][r]", Tuple::v1), shardTask); -} + } /** * Very basic "is it plugged in" style test that indexes a document and @@ -326,40 +338,59 @@ public class TasksIT extends ESIntegTestCase { */ ReentrantLock taskFinishLock = new ReentrantLock(); taskFinishLock.lock(); - CountDownLatch taskRegistered = new CountDownLatch(1); - for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { - ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() { - @Override - public void onTaskRegistered(Task task) { - if (task.getAction().startsWith(IndexAction.NAME)) { - taskRegistered.countDown(); + ListenableActionFuture indexFuture = null; + try { + CountDownLatch taskRegistered = new CountDownLatch(1); + for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { + ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() { + @Override + public void onTaskRegistered(Task task) { + if (task.getAction().startsWith(IndexAction.NAME)) { + taskRegistered.countDown(); + } } - } - @Override - public void onTaskUnregistered(Task task) { - /* - * We can't block all tasks here or the task listing task - * would never return. - */ - if (false == task.getAction().startsWith(IndexAction.NAME)) { - return; + @Override + public void onTaskUnregistered(Task task) { + /* + * We can't block all tasks here or the task listing task + * would never return. + */ + if (false == task.getAction().startsWith(IndexAction.NAME)) { + return; + } + logger.debug("Blocking {} from being unregistered", task); + taskFinishLock.lock(); + taskFinishLock.unlock(); } - logger.debug("Blocking {} from being unregistered", task); - taskFinishLock.lock(); - taskFinishLock.unlock(); - } - }); - } - ListenableActionFuture indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute(); - taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered - ListTasksResponse tasks = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*").setDetailed(true) - .get(); - taskFinishLock.unlock(); - indexFuture.get(); - assertThat(tasks.getTasks(), not(emptyCollectionOf(TaskInfo.class))); - for (TaskInfo task : tasks.getTasks()) { - assertNotNull(task.getStatus()); + }); + } + indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute(); + taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered + + ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*") + .setDetailed(true).get(); + assertThat(listResponse.getTasks(), not(empty())); + for (TaskInfo task : listResponse.getTasks()) { + assertNotNull(task.getStatus()); + GetTaskResponse getResponse = client().admin().cluster().prepareGetTask(task.getTaskId()).get(); + assertFalse("task should still be running", getResponse.getTask().isCompleted()); + TaskInfo fetchedWithGet = getResponse.getTask().getTask(); + assertEquals(task.getId(), fetchedWithGet.getId()); + assertEquals(task.getType(), fetchedWithGet.getType()); + assertEquals(task.getAction(), fetchedWithGet.getAction()); + assertEquals(task.getDescription(), fetchedWithGet.getDescription()); + // The status won't always be equal - it might change between the list and the get. + assertEquals(task.getStartTime(), fetchedWithGet.getStartTime()); + assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos())); + assertEquals(task.isCancellable(), fetchedWithGet.isCancellable()); + assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId()); + } + } finally { + taskFinishLock.unlock(); + if (indexFuture != null) { + indexFuture.get(); + } } } @@ -401,60 +432,91 @@ public class TasksIT extends ESIntegTestCase { .getTasks().size()); } - public void testTasksListWaitForCompletion() throws Exception { + public void testListTasksWaitForCompletion() throws Exception { + waitForCompletionTestCase(id -> { + return client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]") + .setWaitForCompletion(true).execute(); + }, response -> { + assertThat(response.getNodeFailures(), empty()); + assertThat(response.getTaskFailures(), empty()); + }); + } + + public void testGetTaskWaitForCompletion() throws Exception { + waitForCompletionTestCase(id -> { + return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute(); + }, response -> { + // Really we're just happy we didn't get any exceptions + assertNotNull(response.getTask().getTask()); + }); + } + + /** + * Test wait for completion. + * @param wait start waiting for a task. Accepts that id of the task to wait for and returns a future waiting for it. + * @param validator validate the response and return the task ids that were found + */ + private void waitForCompletionTestCase(Function> wait, Consumer validator) + throws Exception { // Start blocking test task ListenableActionFuture future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client()) .execute(); - ListenableActionFuture waitResponseFuture; + ListenableActionFuture waitResponseFuture; + TaskId taskId; try { - // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); + taskId = waitForTestTaskStartOnAllNodes(); // Spin up a request to wait for that task to finish - waitResponseFuture = client().admin().cluster().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).execute(); + waitResponseFuture = wait.apply(taskId); } finally { // Unblock the request so the wait for completion request can finish TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get(); } // Now that the task is unblocked the list response will come back - ListTasksResponse waitResponse = waitResponseFuture.get(); - // If any tasks come back then they are the tasks we asked for - it'd be super weird if this wasn't true - for (TaskInfo task: waitResponse.getTasks()) { - assertEquals(task.getAction(), TestTaskPlugin.TestTaskAction.NAME + "[n]"); - } - // See the next test to cover the timeout case + T waitResponse = waitResponseFuture.get(); + validator.accept(waitResponse); future.get(); } - public void testTasksListWaitForTimeout() throws Exception { + public void testListTasksWaitForTimeout() throws Exception { + waitForTimeoutTestCase(id -> { + ListTasksResponse response = client().admin().cluster().prepareListTasks() + .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100)) + .get(); + assertThat(response.getNodeFailures(), not(empty())); + return response.getNodeFailures(); + }); + } + + public void testGetTaskWaitForTimeout() throws Exception { + waitForTimeoutTestCase(id -> { + Exception e = expectThrows(Exception.class, + () -> client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).setTimeout(timeValueMillis(100)).get()); + return singleton(e); + }); + } + + /** + * Test waiting for a task that times out. + * @param wait wait for the running task and return all the failures you accumulated waiting for it + */ + private void waitForTimeoutTestCase(Function> wait) throws Exception { // Start blocking test task ListenableActionFuture future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client()) .execute(); try { - // Wait for the task to start on all nodes - assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size())); + TaskId taskId = waitForTestTaskStartOnAllNodes(); // Spin up a request that should wait for those tasks to finish // It will timeout because we haven't unblocked the tasks - ListTasksResponse waitResponse = client().admin().cluster().prepareListTasks() - .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100)) - .get(); + Iterable failures = wait.apply(taskId); - assertFalse(waitResponse.getNodeFailures().isEmpty()); - for (FailedNodeException failure : waitResponse.getNodeFailures()) { - Throwable timeoutException = failure.getCause(); - // The exception sometimes comes back wrapped depending on the client - if (timeoutException.getCause() != null) { - timeoutException = timeoutException.getCause(); - } - assertThat(timeoutException, - either(instanceOf(ElasticsearchTimeoutException.class)).or(instanceOf(ReceiveTimeoutTransportException.class))); + for (Throwable failure : failures) { + assertNotNull( + ExceptionsHelper.unwrap(failure, ElasticsearchTimeoutException.class, ReceiveTimeoutTransportException.class)); } } finally { // Now we can unblock those requests @@ -463,6 +525,17 @@ public class TasksIT extends ESIntegTestCase { future.get(); } + private TaskId waitForTestTaskStartOnAllNodes() throws Exception { + AtomicReference result = new AtomicReference<>(); + assertBusy(() -> { + List tasks = client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]") + .get().getTasks(); + assertEquals(internalCluster().size(), tasks.size()); + result.set(tasks.get(0).getTaskId()); + }); + return result.get(); + } + public void testTasksListWaitForNoTask() throws Exception { // Spin up a request to wait for no matching tasks ListenableActionFuture waitResponseFuture = client().admin().cluster().prepareListTasks() @@ -470,7 +543,17 @@ public class TasksIT extends ESIntegTestCase { .execute(); // It should finish quickly and without complaint - assertThat(waitResponseFuture.get().getTasks(), emptyCollectionOf(TaskInfo.class)); + assertThat(waitResponseFuture.get().getTasks(), empty()); + } + + public void testTasksGetWaitForNoTask() throws Exception { + // Spin up a request to wait for no matching tasks + ListenableActionFuture waitResponseFuture = client().admin().cluster().prepareGetTask("notfound:1") + .setWaitForCompletion(true).setTimeout(timeValueMillis(10)) + .execute(); + + // It should finish quickly and without complaint + expectNotFound(() -> waitResponseFuture.get()); } public void testTasksWaitForAllTask() throws Exception { @@ -488,7 +571,7 @@ public class TasksIT extends ESIntegTestCase { // Randomly create an empty index to make sure the type is created automatically if (randomBoolean()) { logger.info("creating an empty results index with custom settings"); - assertAcked(client().admin().indices().prepareCreate(TaskResultsService.TASK_RESULT_INDEX)); + assertAcked(client().admin().indices().prepareCreate(TaskPersistenceService.TASK_INDEX)); } registerTaskManageListeners(TestTaskPlugin.TestTaskAction.NAME); // we need this to get task id of the process @@ -500,34 +583,43 @@ public class TasksIT extends ESIntegTestCase { assertEquals(1, events.size()); TaskInfo taskInfo = events.get(0); - String taskId = taskInfo.getTaskId().toString(); + TaskId taskId = taskInfo.getTaskId(); - GetResponse resultDoc = client().prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, taskId) - .get(); + GetResponse resultDoc = client() + .prepareGet(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE, taskId.toString()).get(); assertTrue(resultDoc.isExists()); Map source = resultDoc.getSource(); + @SuppressWarnings("unchecked") Map task = (Map) source.get("task"); - assertEquals(taskInfo.getNode().getId(), task.get("node")); + assertEquals(taskInfo.getTaskId().getNodeId(), task.get("node")); assertEquals(taskInfo.getAction(), task.get("action")); assertEquals(Long.toString(taskInfo.getId()), task.get("id").toString()); + @SuppressWarnings("unchecked") Map result = (Map) source.get("result"); assertEquals("0", result.get("failure_count").toString()); - assertNoFailures(client().admin().indices().prepareRefresh(TaskResultsService.TASK_RESULT_INDEX).get()); + assertNull(source.get("failure")); - SearchResponse searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX) - .setTypes(TaskResultsService.TASK_RESULT_TYPE) + assertNoFailures(client().admin().indices().prepareRefresh(TaskPersistenceService.TASK_INDEX).get()); + + SearchResponse searchResponse = client().prepareSearch(TaskPersistenceService.TASK_INDEX) + .setTypes(TaskPersistenceService.TASK_TYPE) .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.action", taskInfo.getAction()))) .get(); assertEquals(1L, searchResponse.getHits().totalHits()); - searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX).setTypes(TaskResultsService.TASK_RESULT_TYPE) - .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getNode().getId()))).get(); + searchResponse = client().prepareSearch(TaskPersistenceService.TASK_INDEX).setTypes(TaskPersistenceService.TASK_TYPE) + .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getTaskId().getNodeId()))) + .get(); assertEquals(1L, searchResponse.getHits().totalHits()); + + GetTaskResponse getResponse = expectFinishedTask(taskId); + assertEquals(result, getResponse.getTask().getResultAsMap()); + assertNull(getResponse.getTask().getError()); } public void testTaskFailurePersistence() throws Exception { @@ -545,22 +637,69 @@ public class TasksIT extends ESIntegTestCase { List events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1); assertEquals(1, events.size()); TaskInfo failedTaskInfo = events.get(0); - String failedTaskId = failedTaskInfo.getTaskId().toString(); + TaskId failedTaskId = failedTaskInfo.getTaskId(); GetResponse failedResultDoc = client() - .prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, failedTaskId) + .prepareGet(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE, failedTaskId.toString()) .get(); assertTrue(failedResultDoc.isExists()); Map source = failedResultDoc.getSource(); + @SuppressWarnings("unchecked") Map task = (Map) source.get("task"); - assertEquals(failedTaskInfo.getNode().getId(), task.get("node")); + assertEquals(failedTaskInfo.getTaskId().getNodeId(), task.get("node")); assertEquals(failedTaskInfo.getAction(), task.get("action")); assertEquals(Long.toString(failedTaskInfo.getId()), task.get("id").toString()); + @SuppressWarnings("unchecked") Map error = (Map) source.get("error"); assertEquals("Simulating operation failure", error.get("reason")); assertEquals("illegal_state_exception", error.get("type")); + + assertNull(source.get("result")); + + GetTaskResponse getResponse = expectFinishedTask(failedTaskId); + assertNull(getResponse.getTask().getResult()); + assertEquals(error, getResponse.getTask().getErrorAsMap()); + } + + public void testGetTaskNotFound() throws Exception { + // Node isn't found, tasks index doesn't even exist + expectNotFound(() -> client().admin().cluster().prepareGetTask("not_a_node:1").get()); + + // Node exists but the task still isn't found + expectNotFound(() -> client().admin().cluster().prepareGetTask(new TaskId(internalCluster().getNodeNames()[0], 1)).get()); + } + + public void testNodeNotFoundButTaskFound() throws Exception { + // Save a fake task that looks like it is from a node that isn't part of the cluster + CyclicBarrier b = new CyclicBarrier(2); + TaskPersistenceService resultsService = internalCluster().getInstance(TaskPersistenceService.class); + resultsService.persist( + new PersistedTaskInfo(new TaskInfo(new TaskId("fake", 1), "test", "test", "", null, 0, 0, false, TaskId.EMPTY_TASK_ID), + new RuntimeException("test")), + new ActionListener() { + @Override + public void onResponse(Void response) { + try { + b.await(); + } catch (InterruptedException | BrokenBarrierException e) { + onFailure(e); + } + } + + @Override + public void onFailure(Throwable e) { + throw new RuntimeException(e); + } + }); + b.await(); + + // Now we can find it! + GetTaskResponse response = expectFinishedTask(new TaskId("fake:1")); + assertEquals("test", response.getTask().getTask().getAction()); + assertNotNull(response.getTask().getError()); + assertNull(response.getTask().getResult()); } @Override @@ -638,8 +777,28 @@ public class TasksIT extends ESIntegTestCase { private void assertParentTask(TaskInfo task, TaskInfo parentTask) { assertTrue(task.getParentTaskId().isSet()); - assertEquals(parentTask.getNode().getId(), task.getParentTaskId().getNodeId()); + assertEquals(parentTask.getTaskId().getNodeId(), task.getParentTaskId().getNodeId()); assertTrue(Strings.hasLength(task.getParentTaskId().getNodeId())); assertEquals(parentTask.getId(), task.getParentTaskId().getId()); } + + private ResourceNotFoundException expectNotFound(ThrowingRunnable r) { + Exception e = expectThrows(Exception.class, r); + ResourceNotFoundException notFound = (ResourceNotFoundException) ExceptionsHelper.unwrap(e, ResourceNotFoundException.class); + if (notFound == null) throw new RuntimeException("Expected ResourceNotFoundException", e); + return notFound; + } + + /** + * Fetch the task status from the list tasks API using it's "fallback to get from the task index" behavior. Asserts some obvious stuff + * about the fetched task and returns a map of it's status. + */ + private GetTaskResponse expectFinishedTask(TaskId taskId) throws IOException { + GetTaskResponse response = client().admin().cluster().prepareGetTask(taskId).get(); + assertTrue("the task should have been completed before fetching", response.getTask().isCompleted()); + TaskInfo info = response.getTask().getTask(); + assertEquals(taskId, info.getTaskId()); + assertNull(info.getStatus()); // The test task doesn't have any status + return response; + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 3e996421441..326e909b3ff 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.nodes.BaseNodeRequest; @@ -38,9 +37,8 @@ import org.elasticsearch.action.support.tasks.BaseTasksResponse; import org.elasticsearch.action.support.tasks.TransportTasksAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -52,6 +50,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.tasks.MockTaskManager; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -371,10 +370,10 @@ public class TransportTasksActionTests extends TaskManagerTestCase { assertEquals(testNodes.length, response.getPerNodeTasks().size()); // Coordinating node - assertEquals(2, response.getPerNodeTasks().get(testNodes[0].discoveryNode).size()); + assertEquals(2, response.getPerNodeTasks().get(testNodes[0].discoveryNode.getId()).size()); // Other nodes node for (int i = 1; i < testNodes.length; i++) { - assertEquals(1, response.getPerNodeTasks().get(testNodes[i].discoveryNode).size()); + assertEquals(1, response.getPerNodeTasks().get(testNodes[i].discoveryNode.getId()).size()); } // There should be a single main task when grouped by tasks assertEquals(1, response.getTaskGroups().size()); @@ -387,7 +386,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { listTasksRequest.setActions("testAction[n]"); // only pick node actions response = testNode.transportListTasksAction.execute(listTasksRequest).get(); assertEquals(testNodes.length, response.getPerNodeTasks().size()); - for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { + for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); assertNull(entry.getValue().get(0).getDescription()); } @@ -401,7 +400,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { listTasksRequest.setDetailed(true); // same request only with detailed description response = testNode.transportListTasksAction.execute(listTasksRequest).get(); assertEquals(testNodes.length, response.getPerNodeTasks().size()); - for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { + for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription()); } @@ -438,7 +437,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { listTasksRequest.setActions("testAction"); ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); assertEquals(1, response.getTasks().size()); - String parentNode = response.getTasks().get(0).getNode().getId(); + String parentNode = response.getTasks().get(0).getTaskId().getNodeId(); long parentTaskId = response.getTasks().get(0).getId(); // Find tasks with common parent @@ -493,7 +492,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { listTasksRequest.setActions("testAction[n]"); // only pick node actions ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); assertEquals(testNodes.length, response.getPerNodeTasks().size()); - for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { + for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); assertNull(entry.getValue().get(0).getDescription()); } @@ -503,7 +502,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { listTasksRequest.setDetailed(true); // same request only with detailed description response = testNode.transportListTasksAction.execute(listTasksRequest).get(); assertEquals(testNodes.length, response.getPerNodeTasks().size()); - for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { + for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription()); assertThat(entry.getValue().get(0).getStartTime(), greaterThanOrEqualTo(minimalStartTime)); @@ -739,6 +738,11 @@ public class TransportTasksActionTests extends TaskManagerTestCase { assertEquals(testNodes.length + 1, response.getTasks().size()); // First group by node + DiscoveryNodes.Builder discoNodes = DiscoveryNodes.builder(); + for (TestNode testNode : this.testNodes) { + discoNodes.put(testNode.discoveryNode); + } + response.setDiscoveryNodes(discoNodes.build()); Map byNodes = serialize(response, new ToXContent.MapParams(Collections.singletonMap("group_by", "nodes"))); byNodes = (Map) byNodes.get("nodes"); // One element on the top level diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 06ca3a4f869..b736751b781 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -209,6 +209,11 @@ public class ClusterRerouteRequestTests extends ESTestCase { } builder.endObject(); - return new FakeRestRequest(emptyMap(), params, hasBody ? builder.bytes() : null); + FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(); + requestBuilder.withParams(params); + if (hasBody) { + requestBuilder.withContent(builder.bytes()); + } + return requestBuilder.build(); } } diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index 2df6f798b8e..2ae23774bdf 100644 --- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -22,8 +22,6 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -37,7 +35,6 @@ import org.elasticsearch.test.StreamsUtils; import java.io.IOException; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -167,6 +164,13 @@ public class MultiSearchRequestTests extends ESTestCase { builder.string()); } + public void testMaxConcurrentSearchRequests() { + MultiSearchRequest request = new MultiSearchRequest(); + request.maxConcurrentSearchRequests(randomIntBetween(1, Integer.MAX_VALUE)); + expectThrows(IllegalArgumentException.class, () -> + request.maxConcurrentSearchRequests(randomIntBetween(Integer.MIN_VALUE, 0))); + } + private IndicesQueriesRegistry registry() { IndicesQueriesRegistry registry = new IndicesQueriesRegistry(); QueryParser parser = MatchAllQueryBuilder::fromXContent; diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java new file mode 100644 index 00000000000..d751424ef72 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilter; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.LocalTransportAddress; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportMultiSearchActionTests extends ESTestCase { + + public void testBatchExecute() throws Exception { + // Initialize depedencies of TransportMultiSearchAction + Settings settings = Settings.builder() + .put("node.name", TransportMultiSearchActionTests.class.getSimpleName()) + .build(); + ActionFilters actionFilters = mock(ActionFilters.class); + when(actionFilters.filters()).thenReturn(new ActionFilter[0]); + ThreadPool threadPool = new ThreadPool(settings); + TaskManager taskManager = mock(TaskManager.class); + TransportService transportService = mock(TransportService.class); + when(transportService.getTaskManager()).thenReturn(taskManager); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("test")).build()); + IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY); + + // Keep track of the number of concurrent searches started by multi search api, + // and if there are more searches than is allowed create an error and remember that. + int maxAllowedConcurrentSearches = scaledRandomIntBetween(1, 20); + AtomicInteger counter = new AtomicInteger(); + AtomicReference errorHolder = new AtomicReference<>(); + TransportAction searchAction = new TransportAction + (Settings.EMPTY, "action", threadPool, actionFilters, resolver, taskManager) { + @Override + protected void doExecute(SearchRequest request, ActionListener listener) { + int currentConcurrentSearches = counter.incrementAndGet(); + if (currentConcurrentSearches > maxAllowedConcurrentSearches) { + errorHolder.set(new AssertionError("Current concurrent search [" + currentConcurrentSearches + + "] is higher than is allowed [" + maxAllowedConcurrentSearches + "]")); + } + threadPool.executor(ThreadPool.Names.GENERIC).execute( + () -> { + try { + Thread.sleep(scaledRandomIntBetween(10, 1000)); + } catch (InterruptedException e) { + } + counter.decrementAndGet(); + listener.onResponse(new SearchResponse()); + } + ); + } + }; + TransportMultiSearchAction action = + new TransportMultiSearchAction(threadPool, actionFilters, transportService, clusterService, searchAction, resolver, 10); + + // Execute the multi search api and fail if we find an error after executing: + try { + int numSearchRequests = randomIntBetween(16, 128); + MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + multiSearchRequest.maxConcurrentSearchRequests(maxAllowedConcurrentSearches); + for (int i = 0; i < numSearchRequests; i++) { + multiSearchRequest.add(new SearchRequest()); + } + + MultiSearchResponse response = action.execute(multiSearchRequest).actionGet(); + assertThat(response.getResponses().length, equalTo(numSearchRequests)); + assertThat(errorHolder.get(), nullValue()); + } finally { + assertTrue(ESTestCase.terminate(threadPool)); + } + } + + public void testDefaultMaxConcurrentSearches() { + int numDataNodes = randomIntBetween(1, 10); + DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); + for (int i = 0; i < numDataNodes; i++) { + builder.put(new DiscoveryNode("_id" + i, new LocalTransportAddress("_id" + i), Collections.emptyMap(), + Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT)); + } + builder.put(new DiscoveryNode("master", new LocalTransportAddress("mater"), Collections.emptyMap(), + Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT)); + builder.put(new DiscoveryNode("ingest", new LocalTransportAddress("ingest"), Collections.emptyMap(), + Collections.singleton(DiscoveryNode.Role.INGEST), Version.CURRENT)); + + ClusterState state = ClusterState.builder(new ClusterName("_name")).nodes(builder).build(); + int result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state); + assertThat(result, equalTo(10 * numDataNodes)); + + state = ClusterState.builder(new ClusterName("_name")).build(); + result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state); + assertThat(result, equalTo(1)); + } + +} diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java new file mode 100644 index 00000000000..be4a7b29703 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.master; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.ClusterState; + +public class TransportMasterNodeActionUtils { + + /** + * Allows to directly call {@link TransportMasterNodeAction#masterOperation(MasterNodeRequest, ClusterState, ActionListener)} which is + * a protected method. + */ + public static , Response extends ActionResponse> void runMasterOperation( + TransportMasterNodeAction masterNodeAction, Request request, ClusterState clusterState, + ActionListener actionListener) throws Exception { + assert masterNodeAction.checkBlock(request, clusterState) == null; + masterNodeAction.masterOperation(request, clusterState, actionListener); + } +} diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java index d2ef349625e..a6e74a47706 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java @@ -72,4 +72,8 @@ public class JavaVersionTests extends ESTestCase { assertFalse(JavaVersion.isValid(version)); } } + + public void testJava8Compat() { + assertEquals(JavaVersion.parse("1.8"), JavaVersion.parse("8")); + } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 419104bfe34..494aa7d1095 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -1,4 +1,3 @@ -/* /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -46,6 +45,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.SortedSet; @@ -127,6 +127,44 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { } } + public void testRestoreSnapshotWithMissingChecksum() throws Exception { + final String repo = "test_repo"; + final String snapshot = "test_1"; + final String indexName = "index-2.3.4"; + final String repoFileId = "missing-checksum-repo-2.3.4"; + Path repoFile = getBwcIndicesPath().resolve(repoFileId + ".zip"); + URI repoFileUri = repoFile.toUri(); + URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/"); + logger.info("--> creating repository [{}] for repo file [{}]", repo, repoFileId); + assertAcked(client().admin().cluster().preparePutRepository(repo) + .setType("url") + .setSettings(Settings.builder().put("url", repoJarUri.toString()))); + + logger.info("--> get snapshot and check its indices"); + GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo).setSnapshots(snapshot).get(); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); + SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0); + assertThat(snapshotInfo.indices(), equalTo(Arrays.asList(indexName))); + + logger.info("--> restoring snapshot"); + RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + RestoreInfo restoreInfo = response.getRestoreInfo(); + assertThat(restoreInfo.successfulShards(), greaterThan(0)); + assertThat(restoreInfo.successfulShards(), equalTo(restoreInfo.totalShards())); + assertThat(restoreInfo.failedShards(), equalTo(0)); + String index = restoreInfo.indices().get(0); + assertThat(index, equalTo(indexName)); + + logger.info("--> check search"); + SearchResponse searchResponse = client().prepareSearch(index).get(); + assertThat(searchResponse.getHits().totalHits(), greaterThan(0L)); + + logger.info("--> cleanup"); + cluster().wipeIndices(restoreInfo.indices().toArray(new String[restoreInfo.indices().size()])); + cluster().wipeTemplates(); + } + private List repoVersions() throws Exception { return listRepoVersions("repo"); } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java index 731ecb859ee..72748a59986 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java @@ -140,24 +140,24 @@ public class ClusterChangedEventTests extends ESTestCase { */ public void testIndexMetaDataChange() { final int numNodesInCluster = 3; - final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices); - final ClusterState newState = originalState; // doesn't matter for this test, just need a non-null value - final ClusterChangedEvent event = new ClusterChangedEvent("_na_", originalState, newState); + final ClusterState state = createState(numNodesInCluster, randomBoolean(), initialIndices); // test when its not the same IndexMetaData final Index index = initialIndices.get(0); - final IndexMetaData originalIndexMeta = originalState.metaData().index(index); + final IndexMetaData originalIndexMeta = state.metaData().index(index); // make sure the metadata is actually on the cluster state assertNotNull("IndexMetaData for " + index + " should exist on the cluster state", originalIndexMeta); IndexMetaData newIndexMeta = createIndexMetadata(index, originalIndexMeta.getVersion() + 1); - assertTrue("IndexMetaData with different version numbers must be considered changed", event.indexMetaDataChanged(newIndexMeta)); + assertTrue("IndexMetaData with different version numbers must be considered changed", + ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, newIndexMeta)); // test when it doesn't exist newIndexMeta = createIndexMetadata(new Index("doesntexist", UUIDs.randomBase64UUID())); - assertTrue("IndexMetaData that didn't previously exist should be considered changed", event.indexMetaDataChanged(newIndexMeta)); + assertTrue("IndexMetaData that didn't previously exist should be considered changed", + ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, newIndexMeta)); // test when its the same IndexMetaData - assertFalse("IndexMetaData should be the same", event.indexMetaDataChanged(originalIndexMeta)); + assertFalse("IndexMetaData should be the same", ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, originalIndexMeta)); } /** diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index e515e0f9bf4..0127702c5ea 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -18,16 +18,31 @@ */ package org.elasticsearch.cluster.routing; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; +import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + public class OperationRoutingTests extends ESTestCase{ public void testGenerateShardId() { @@ -170,4 +185,48 @@ public class OperationRoutingTests extends ESTestCase{ assertEquals(shard, entry.getValue().intValue()); } } + + public void testPreferNodes() throws InterruptedException, IOException { + TestThreadPool threadPool = null; + ClusterService clusterService = null; + try { + threadPool = new TestThreadPool("testPreferNodes"); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + final String indexName = "test"; + ClusterServiceUtils.setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(indexName, true, randomInt(8))); + final Index index = clusterService.state().metaData().index(indexName).getIndex(); + final List shards = clusterService.state().getRoutingNodes().assignedShards(new ShardId(index, 0)); + final int count = randomIntBetween(1, shards.size()); + int position = 0; + final List nodes = new ArrayList<>(); + final List expected = new ArrayList<>(); + for (int i = 0; i < count; i++) { + if (randomBoolean() && !shards.get(position).initializing()) { + nodes.add(shards.get(position).currentNodeId()); + expected.add(shards.get(position)); + position++; + } else { + nodes.add("missing_" + i); + } + } + final ShardIterator it = + new OperationRouting(Settings.EMPTY, new AwarenessAllocationDecider()) + .getShards(clusterService.state(), indexName, 0, "_prefer_nodes:" + String.join(",", nodes)); + final List all = new ArrayList<>(); + ShardRouting shard; + while ((shard = it.nextOrNull()) != null) { + all.add(shard); + } + final Set preferred = new HashSet<>(); + preferred.addAll(all.subList(0, expected.size())); + // the preferred shards should be at the front of the list + assertThat(preferred, containsInAnyOrder(expected.toArray())); + // verify all the shards are there + assertThat(all.size(), equalTo(shards.size())); + } finally { + IOUtils.close(clusterService); + terminate(threadPool); + } + } + } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java index 5ff4a328ef6..d8dedb1e8e8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java @@ -62,7 +62,7 @@ public class RoutingBackwardCompatibilityTests extends ESTestCase { MetaData.Builder metaData = MetaData.builder().put(indexMetaData, false); RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); - final int shardId = operationRouting.indexShards(clusterState, index, type, id, routing).shardId().getId(); + final int shardId = operationRouting.indexShards(clusterState, index, id, routing).shardId().getId(); assertEquals(currentExpectedShard, shardId); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java index 307df91c302..1f39706e4f4 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java @@ -161,7 +161,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase { } } - private static final class RandomAllocationDecider extends AllocationDecider { + public static final class RandomAllocationDecider extends AllocationDecider { private final Random random; diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index c9dd0e26361..e9ec914d0dc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.test.ESAllocationTestCase; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import static java.util.Collections.singletonMap; @@ -401,15 +402,23 @@ public class RoutingIteratorTests extends ESAllocationTestCase { assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0)); assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId))); - shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_node:node1"); + shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1"); assertThat(shardIterators.size(), equalTo(1)); assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0)); assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1")); - shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_node:node1"); + shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1,node2"); assertThat(shardIterators.size(), equalTo(1)); - assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0)); - assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1")); + Iterator iterator = shardIterators.iterator(); + final ShardIterator it = iterator.next(); + assertThat(it.shardId().id(), equalTo(0)); + final String firstNodeId = it.nextOrNull().currentNodeId(); + assertThat(firstNodeId, anyOf(equalTo("node1"), equalTo("node2"))); + if ("node1".equals(firstNodeId)) { + assertThat(it.nextOrNull().currentNodeId(), equalTo("node2")); + } else { + assertThat(it.nextOrNull().currentNodeId(), equalTo("node1")); + } } public void testReplicaShardPreferenceIters() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java index 49ba26231da..6302f9c67b1 100644 --- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java +++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java @@ -309,24 +309,54 @@ public class TimeZoneRoundingTests extends ESTestCase { assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20))); } + /** + * Special test for intervals that don't fit evenly into rounding interval. + * In this case, when interval crosses DST transition point, rounding in local + * time can land in a DST gap which results in wrong UTC rounding values. + */ + public void testIntervalRounding_NotDivisibleInteval() { + DateTimeZone tz = DateTimeZone.forID("CET"); + long interval = TimeUnit.MINUTES.toMillis(14); + TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz); + + assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), equalTo(time("2016-03-27T01:30:00+01:00"))); + assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), equalTo(time("2016-03-27T01:44:00+01:00"))); + assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), equalTo(time("2016-03-27T01:58:00+01:00"))); + assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), equalTo(time("2016-03-27T03:00:00+02:00"))); + assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), equalTo(time("2016-03-27T03:08:00+02:00"))); + assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), equalTo(time("2016-03-27T03:22:00+02:00"))); + assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), equalTo(time("2016-03-27T03:36:00+02:00"))); + } + /** * randomized test on {@link TimeIntervalRounding} with random interval and time zone offsets */ public void testIntervalRoundingRandom() { - for (int i = 0; i < 1000; ++i) { - // max random interval is a year, can be negative - long interval = Math.abs(randomLong() % (TimeUnit.DAYS.toMillis(365))); - TimeZoneRounding rounding; - int timezoneOffset = randomIntBetween(-23, 23); - rounding = new TimeZoneRounding.TimeIntervalRounding(interval, DateTimeZone.forOffsetHours(timezoneOffset)); - long date = Math.abs(randomLong() % ((long) 10e11)); - final long roundedDate = rounding.round(date); - final long nextRoundingValue = rounding.nextRoundingValue(roundedDate); - assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate))); - assertThat("Rounded value smaller or equal than unrounded, regardless of timezone", roundedDate, lessThanOrEqualTo(date)); - assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate)); - assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate, equalTo(interval)); - assertThat("NextRounding value should be a rounded date", nextRoundingValue, equalTo(rounding.round(nextRoundingValue))); + for (int i = 0; i < 1000; i++) { + TimeUnit unit = randomFrom(new TimeUnit[] {TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS}); + long interval = unit.toMillis(randomIntBetween(1, 365)); + DateTimeZone tz = randomDateTimeZone(); + TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz); + long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00 + try { + final long roundedDate = rounding.round(date); + final long nextRoundingValue = rounding.nextRoundingValue(roundedDate); + assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate))); + assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date)); + assertThat("Values smaller than rounded value should round further down", rounding.round(roundedDate - 1), + lessThan(roundedDate)); + + if (tz.isFixed()) { + assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate)); + assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate, + equalTo(interval)); + assertThat("NextRounding value should be a rounded date", nextRoundingValue, + equalTo(rounding.round(nextRoundingValue))); + } + } catch (AssertionError e) { + logger.error("Rounding error at {}, timezone {}, interval: {},", new DateTime(date, tz), tz, interval); + throw e; + } } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index 3cd68bea038..cc36625e68f 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -28,6 +28,8 @@ import org.joda.time.PeriodType; import java.io.IOException; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; +import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; @@ -123,20 +125,22 @@ public class TimeValueTests extends ESTestCase { TimeValue.parseTimeValue("10W", null, "test")); } - private void assertEqualityAfterSerialize(TimeValue value) throws IOException { + private void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException { BytesStreamOutput out = new BytesStreamOutput(); value.writeTo(out); + assertEquals(expectedSize, out.size()); StreamInput in = StreamInput.wrap(out.bytes()); - TimeValue inValue = TimeValue.readTimeValue(in); + TimeValue inValue = new TimeValue(in); assertThat(inValue, equalTo(value)); } public void testSerialize() throws Exception { - assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS)); - assertEqualityAfterSerialize(new TimeValue(-1)); - assertEqualityAfterSerialize(new TimeValue(1, TimeUnit.NANOSECONDS)); + assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS), 8); + assertEqualityAfterSerialize(timeValueNanos(-1), 1); + assertEqualityAfterSerialize(timeValueNanos(1), 1); + assertEqualityAfterSerialize(timeValueSeconds(30), 6); } public void testFailOnUnknownUnits() { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java index b4ddd4ab589..461428581c5 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java @@ -27,8 +27,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; - public abstract class BaseXContentTestCase extends ESTestCase { public abstract XContentType xcontentType(); @@ -136,5 +134,26 @@ public abstract class BaseXContentTestCase extends ESTestCase { assertEquals(Token.VALUE_NULL, parser.nextToken()); assertEquals(Token.END_OBJECT, parser.nextToken()); assertNull(parser.nextToken()); + + os = new ByteArrayOutputStream(); + try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) { + generator.writeStartObject(); + generator.writeFieldName("test"); + generator.writeRawValue(new BytesArray(rawData)); + generator.writeEndObject(); + } + + parser = xcontentType().xContent().createParser(os.toByteArray()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("test", parser.currentName()); + assertEquals(Token.START_OBJECT, parser.nextToken()); + assertEquals(Token.FIELD_NAME, parser.nextToken()); + assertEquals("foo", parser.currentName()); + assertEquals(Token.VALUE_NULL, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertEquals(Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } } diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java index d04bc28936c..454c973cc34 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java @@ -20,14 +20,13 @@ package org.elasticsearch.fieldstats; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.fieldstats.FieldStats; +import org.elasticsearch.action.fieldstats.FieldStatsAction; import org.elasticsearch.action.fieldstats.FieldStatsResponse; import org.elasticsearch.action.fieldstats.IndexConstraint; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.cache.request.RequestCacheStats; import org.elasticsearch.test.ESIntegTestCase; import java.util.ArrayList; @@ -40,11 +39,12 @@ import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX; import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.nullValue; /** + * Tests for the {@link FieldStatsAction}. */ public class FieldStatsIntegrationIT extends ESIntegTestCase { @@ -150,7 +150,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { .setFields("byte", "short", "integer", "long", "float", "double", "string").get(); assertAllSuccessful(response); - for (FieldStats stats : response.getAllFieldStats().values()) { + for (FieldStats stats : response.getAllFieldStats().values()) { assertThat(stats.getMaxDoc(), equalTo((long) numDocs)); assertThat(stats.getDocCount(), equalTo((long) numDocs)); assertThat(stats.getDensity(), equalTo(100)); @@ -462,6 +462,38 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase { assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMaxValue(), equalTo(100L)); } + public void testCached() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test").setSettings("index.number_of_replicas", 0)); + indexRange("test", "value", 0, 99); + + // First query should be a cache miss + FieldStatsResponse fieldStats = client().prepareFieldStats().setFields("value").get(); + assertEquals(100, fieldStats.getAllFieldStats().get("value").getDocCount()); + RequestCacheStats indexStats = client().admin().indices().prepareStats().get().getIndex("test").getTotal().getRequestCache(); + assertEquals(0, indexStats.getHitCount()); + assertThat(indexStats.getMemorySizeInBytes(), greaterThan(0L)); + + // Second query should be a cache hit + fieldStats = client().prepareFieldStats().setFields("value").get(); + assertEquals(100, fieldStats.getAllFieldStats().get("value").getDocCount()); + indexStats = client().admin().indices().prepareStats().get().getIndex("test").getTotal().getRequestCache(); + assertThat(indexStats.getHitCount(), greaterThan(0L)); + assertThat(indexStats.getMemorySizeInBytes(), greaterThan(0L)); + + // Indexing some new documents and refreshing should give you consistent data. + long oldHitCount = indexStats.getHitCount(); + indexRange("test", "value", 100, 199); + fieldStats = client().prepareFieldStats().setFields("value").get(); + assertEquals(200, fieldStats.getAllFieldStats().get("value").getDocCount()); + // Because we refreshed the index we don't have any more hits in the cache. This is read from the index. + assertEquals(oldHitCount, indexStats.getHitCount()); + + // We can also turn off the cache entirely + fieldStats = client().prepareFieldStats().setFields("value").get(); + assertEquals(200, fieldStats.getAllFieldStats().get("value").getDocCount()); + assertEquals(oldHitCount, indexStats.getHitCount()); + } + private void indexRange(String index, long from, long to) throws Exception { indexRange(index, "value", from, to); } diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java index 6af06f605dc..264876b7963 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java @@ -94,10 +94,22 @@ public class NettyHttpClient implements Closeable { @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. public final Collection post(SocketAddress remoteAddress, Tuple... urisAndBodies) throws InterruptedException { + return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + public final Collection put(SocketAddress remoteAddress, Tuple... urisAndBodies) + throws InterruptedException { + return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies); + } + + @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods. + private final Collection processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple... urisAndBodies) throws InterruptedException { Collection requests = new ArrayList<>(urisAndBodies.length); for (Tuple uriAndBody : urisAndBodies) { ChannelBuffer content = ChannelBuffers.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8); - HttpRequest request = new DefaultHttpRequest(HTTP_1_1, HttpMethod.POST, uriAndBody.v1()); + HttpRequest request = new DefaultHttpRequest(HTTP_1_1, method, uriAndBody.v1()); request.headers().add(HOST, "localhost"); request.headers().add(HttpHeaders.Names.CONTENT_LENGTH, content.readableBytes()); request.setContent(content); diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java index ba6b4438aaa..8fdd02f8624 100644 --- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java +++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java @@ -29,12 +29,12 @@ import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import java.util.Collection; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; @@ -54,7 +54,6 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase { .build(); } - @TestLogging("_root:DEBUG,org.elasticsearch.common.breaker:TRACE,org.elasticsearch.test:TRACE,org.elasticsearch.transport:TRACE") public void testLimitsInFlightRequests() throws Exception { ensureGreen(); @@ -90,9 +89,36 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase { } } - private void assertAtLeastOnceExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { - long countResponseErrors = responses.stream().filter(r -> r.getStatus().equals(expectedStatus)).count(); - assertThat(countResponseErrors, greaterThan(0L)); + @AwaitsFix(bugUrl = "muted while investigating") + public void testDoesNotLimitExcludedRequests() throws Exception { + ensureGreen(); + @SuppressWarnings("unchecked") + Tuple[] requestUris = new Tuple[1500]; + for (int i = 0; i < requestUris.length; i++) { + requestUris[i] = Tuple.tuple("/_cluster/settings", + "{ \"transient\": {\"indices.ttl.interval\": \"40s\" } }"); + } + + HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class); + InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress + ().boundAddresses()); + + try (NettyHttpClient nettyHttpClient = new NettyHttpClient()) { + Collection responses = nettyHttpClient.put(inetSocketTransportAddress.address(), requestUris); + assertThat(responses, hasSize(requestUris.length)); + assertAllInExpectedStatus(responses, HttpResponseStatus.OK); + } + } + + private void assertAtLeastOnceExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countExpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus)).count(); + assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L)); + } + + private void assertAllInExpectedStatus(Collection responses, HttpResponseStatus expectedStatus) { + long countUnexpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus) == false).count(); + assertThat("Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus + + "] requests had a different one", countUnexpectedStatus, equalTo(0L)); } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java index 3c791e72b5f..2b2c9288f17 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java @@ -65,4 +65,22 @@ public class CharFilterTests extends ESTokenStreamTestCase { // Repeat one more time to make sure that char filter is reinitialized correctly assertTokenStreamContents(analyzer1.tokenStream("test", "hello!"), new String[]{"hello"}); } + + public void testPatternReplaceCharFilter() throws Exception { + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.char_filter.my_mapping.type", "pattern_replace") + .put("index.analysis.char_filter.my_mapping.pattern", "ab*") + .put("index.analysis.char_filter.my_mapping.replacement", "oo") + .put("index.analysis.char_filter.my_mapping.flags", "CASE_INSENSITIVE") + .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard") + .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "my_mapping") + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); + NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter"); + + assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"}); + } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java index 062774bf2f7..4e4d638d355 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -22,9 +22,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.spatial.util.GeoUtils; import org.elasticsearch.Version; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint; import static org.hamcrest.Matchers.allOf; @@ -105,8 +105,8 @@ public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImpl assertThat(docCount, greaterThan(0)); for (int i = 0; i < docCount; ++i) { final GeoPoint point = values.valueAt(i); - assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT_INCL), lessThanOrEqualTo(GeoUtils.MAX_LAT_INCL))); - assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON_INCL), lessThanOrEqualTo(GeoUtils.MAX_LON_INCL))); + assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT), lessThanOrEqualTo(GeoUtils.MAX_LAT))); + assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON), lessThanOrEqualTo(GeoUtils.MAX_LON))); } } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java index 860c66863ff..7a800d469a1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java @@ -26,7 +26,7 @@ import static org.hamcrest.Matchers.equalTo; public class UidTests extends ESTestCase { public void testCreateAndSplitId() { BytesRef createUid = Uid.createUidAsBytes("foo", "bar"); - BytesRef[] splitUidIntoTypeAndId = Uid.splitUidIntoTypeAndId(createUid); + BytesRef[] splitUidIntoTypeAndId = splitUidIntoTypeAndId(createUid); assertThat("foo", equalTo(splitUidIntoTypeAndId[0].utf8ToString())); assertThat("bar", equalTo(splitUidIntoTypeAndId[1].utf8ToString())); // split also with an offset @@ -34,8 +34,29 @@ public class UidTests extends ESTestCase { ref.offset = 9; ref.length = createUid.length; System.arraycopy(createUid.bytes, createUid.offset, ref.bytes, ref.offset, ref.length); - splitUidIntoTypeAndId = Uid.splitUidIntoTypeAndId(ref); + splitUidIntoTypeAndId = splitUidIntoTypeAndId(ref); assertThat("foo", equalTo(splitUidIntoTypeAndId[0].utf8ToString())); assertThat("bar", equalTo(splitUidIntoTypeAndId[1].utf8ToString())); } + + public static BytesRef[] splitUidIntoTypeAndId(BytesRef uid) { + int loc = -1; + final int limit = uid.offset + uid.length; + for (int i = uid.offset; i < limit; i++) { + if (uid.bytes[i] == Uid.DELIMITER_BYTE) { // 0x23 is equal to '#' + loc = i; + break; + } + } + + if (loc == -1) { + return null; + } + + int idStart = loc + 1; + return new BytesRef[] { + new BytesRef(uid.bytes, uid.offset, loc - uid.offset), + new BytesRef(uid.bytes, idStart, limit - idStart) + }; + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java index ee19d094a3f..a3909637548 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java @@ -302,4 +302,19 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase { assertEquals(1457654400000L, dvField.numericValue().longValue()); assertFalse(dvField.fieldType().stored()); } + + public void testNullConfigValuesFail() throws MapperParsingException, IOException { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "date") + .field("format", (String) null) + .endObject() + .endObject() + .endObject().endObject().string(); + + Exception e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping))); + assertEquals("[format] must not have a [null] value", e.getMessage()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java index de14f38d6a9..224d512cb53 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType; import org.elasticsearch.index.mapper.ParsedDocument; @@ -458,4 +459,19 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase { assertThat(fieldType.fielddataMaxFrequency(), equalTo((double) Integer.MAX_VALUE)); assertThat(fieldType.fielddataMinSegmentSize(), equalTo(1000)); } + + public void testNullConfigValuesFail() throws MapperParsingException, IOException { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("analyzer", (String) null) + .endObject() + .endObject() + .endObject().endObject().string(); + + Exception e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping))); + assertEquals("[analyzer] must not have a [null] value", e.getMessage()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java index 177d3b7b0f7..b2e1989454c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.mapper.externalvalues; -import org.apache.lucene.spatial.util.GeoEncodingUtils; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.compress.CompressedXContent; @@ -88,7 +88,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0))); + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0))); } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); @@ -146,7 +146,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0))); + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0))); } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); @@ -208,7 +208,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0))); + assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0))); } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java index a1fdb7ec60f..202afd7a4b1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.geo; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -46,7 +47,6 @@ import java.util.List; import java.util.Map; import java.lang.NumberFormatException; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonHash; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -90,7 +90,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (indexCreatedBefore22 == true) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -197,7 +197,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("89.0,1.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(89.0, 1.0))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(89.0, 1.0))); } doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -209,7 +209,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("-89.0,-1.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(-89.0, -1.0))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(-89.0, -1.0))); } doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -221,7 +221,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("-1.0,-179.0")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(-1.0, -179.0))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(-1.0, -179.0))); } } @@ -408,7 +408,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -441,7 +441,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3)); // indexed hash - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); // point field for 2nd value assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4)); @@ -450,7 +450,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5)); // indexed hash - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5))); } else { assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2)); assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2)); @@ -459,14 +459,14 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5)); if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5))); } } } @@ -491,7 +491,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -517,7 +517,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -559,12 +559,12 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5))); } } @@ -588,7 +588,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -613,7 +613,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -639,7 +639,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -669,14 +669,14 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5)); if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5))); } } else { assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(4)); @@ -685,12 +685,12 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase { assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2)); assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3)); assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3)); - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4)); assertThat(doc.rootDoc().getFields("point.lon")[2].numericValue().doubleValue(), equalTo(1.5)); assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5)); - assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5))); + assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5))); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java index 837cef6a17c..90528c9a8f4 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.geo; +import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.compress.CompressedXContent; @@ -35,7 +36,6 @@ import org.elasticsearch.test.VersionUtils; import java.util.Collection; import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonHash; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -72,7 +72,7 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } @@ -96,7 +96,7 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase { if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3")); } else { - assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3))); + assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3))); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java index 522a35ccd5d..884f52cc0ed 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.ip; import java.net.InetAddress; import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.document.XInetAddressPoint; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.util.BytesRef; @@ -69,11 +68,11 @@ public class IpFieldTypeTests extends FieldTypeTestCase { ip = "2001:db8::2:1"; String prefix = ip + "/64"; - assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null)); + assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null)); ip = "192.168.1.7"; prefix = ip + "/16"; - assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null)); + assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null)); ft.setIndexOptions(IndexOptions.NONE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -88,7 +87,7 @@ public class IpFieldTypeTests extends FieldTypeTestCase { assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), - XInetAddressPoint.MAX_VALUE), + InetAddressPoint.MAX_VALUE), ft.rangeQuery(null, null, randomBoolean(), randomBoolean())); assertEquals( @@ -106,13 +105,13 @@ public class IpFieldTypeTests extends FieldTypeTestCase { assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), - XInetAddressPoint.MAX_VALUE), + InetAddressPoint.MAX_VALUE), ft.rangeQuery("2001:db8::", null, true, randomBoolean())); assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), - XInetAddressPoint.MAX_VALUE), + InetAddressPoint.MAX_VALUE), ft.rangeQuery("2001:db8::", null, false, randomBoolean())); assertEquals( @@ -152,7 +151,7 @@ public class IpFieldTypeTests extends FieldTypeTestCase { assertEquals( InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::1:0:0:0"), - XInetAddressPoint.MAX_VALUE), + InetAddressPoint.MAX_VALUE), // same lo/hi values but inclusive=false so this won't match anything ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true)); diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java index 7780d218b52..387df7ac3ca 100644 --- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java @@ -22,10 +22,10 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery; -import org.apache.lucene.spatial.util.GeoEncodingUtils; import org.elasticsearch.Version; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery; import org.elasticsearch.test.AbstractQueryTestCase; @@ -213,7 +213,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase queryBuilderPoints = queryBuilder.points(); - double[] lats = geoQuery.getLats(); - double[] lons = geoQuery.getLons(); + assertEquals(1, geoQuery.getPolygons().length); + double[] lats = geoQuery.getPolygons()[0].getPolyLats(); + double[] lons = geoQuery.getPolygons()[0].getPolyLons(); assertThat(lats.length, equalTo(queryBuilderPoints.size())); assertThat(lons.length, equalTo(queryBuilderPoints.size())); for (int i=0; i < queryBuilderPoints.size(); ++i) { @@ -321,8 +322,9 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase mappingConsumer = (type, mapping) -> { @@ -1575,7 +1565,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { } } routing = ShardRoutingHelper.moveToStarted(routing); - newShard.updateRoutingEntry(routing, true); + newShard.updateRoutingEntry(routing); assertHitCount(client().prepareSearch("index_1").get(), 2); } // now check that it's persistent ie. that the added shards are committed @@ -1587,7 +1577,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode)); assertTrue(newShard.recoverFromStore()); routing = ShardRoutingHelper.moveToStarted(routing); - newShard.updateRoutingEntry(routing, true); + newShard.updateRoutingEntry(routing); assertHitCount(client().prepareSearch("index_1").get(), 2); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 2543668f557..4938f686f60 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -155,10 +155,12 @@ public class RefreshListenersTests extends ESTestCase { DummyRefreshListener forcingListener = new DummyRefreshListener(); listeners.addOrNotify(index.getTranslogLocation(), forcingListener); assertTrue("Forced listener wasn't forced?", forcingListener.forcedRefresh.get()); + forcingListener.assertNoError(); // That forces all the listeners through. It would be on the listener ThreadPool but we've made all of those execute immediately. for (DummyRefreshListener listener : nonForcedListeners) { assertEquals("Expected listener called with unforced refresh!", Boolean.FALSE, listener.forcedRefresh.get()); + listener.assertNoError(); } assertFalse(listeners.refreshNeeded()); } @@ -174,8 +176,9 @@ public class RefreshListenersTests extends ESTestCase { } DummyRefreshListener listener = new DummyRefreshListener(); - listeners.addOrNotify(index.getTranslogLocation(), listener); + assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener)); assertFalse(listener.forcedRefresh.get()); + listener.assertNoError(); } /** @@ -192,13 +195,17 @@ public class RefreshListenersTests extends ESTestCase { }); refresher.start(); try { - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 1000; i++) { Engine.Index index = index("1"); - DummyRefreshListener listener = new DummyRefreshListener(); - listeners.addOrNotify(index.getTranslogLocation(), listener); - assertBusy(() -> assertNotNull(listener.forcedRefresh.get())); + boolean immediate = listeners.addOrNotify(index.getTranslogLocation(), listener); + if (immediate) { + assertNotNull(listener.forcedRefresh.get()); + } else { + assertBusy(() -> assertNotNull(listener.forcedRefresh.get())); + } assertFalse(listener.forcedRefresh.get()); + listener.assertNoError(); } } finally { run.set(false); @@ -234,6 +241,7 @@ public class RefreshListenersTests extends ESTestCase { if (threadCount < maxListeners) { assertFalse(listener.forcedRefresh.get()); } + listener.assertNoError(); Engine.Get get = new Engine.Get(false, index.uid()); try (Engine.GetResult getResult = engine.get(get)) { @@ -281,13 +289,24 @@ public class RefreshListenersTests extends ESTestCase { /** * When the listener is called this captures it's only argument. */ - private AtomicReference forcedRefresh = new AtomicReference<>(); + AtomicReference forcedRefresh = new AtomicReference<>(); + private volatile Throwable error; @Override public void accept(Boolean forcedRefresh) { - assertNotNull(forcedRefresh); - Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh); - assertNull("Listener called twice", oldValue); + try { + assertNotNull(forcedRefresh); + Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh); + assertNull("Listener called twice", oldValue); + } catch (Throwable e) { + error = e; + } + } + + public void assertNoError() { + if (error != null) { + throw new RuntimeException(error); + } } } } diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java index ffb64f991cc..f31733dc477 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java @@ -31,7 +31,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.Version; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESTestCase; @@ -74,11 +73,9 @@ public class StoreRecoveryTests extends ESTestCase { assertEquals(numFiles, targetNumFiles); assertEquals(indexStats.totalFileCount(), targetNumFiles); if (hardLinksSupported(createTempDir())) { - assertEquals("upgrade to HardlinkCopyDirectoryWrapper in Lucene 6.1", Version.LATEST, Version.LUCENE_6_0_1); - // assertEquals(indexStats.reusedFileCount(), targetNumFiles); -- uncomment this once upgraded to Lucene 6.1 - assertEquals(indexStats.reusedFileCount(), 0); + assertEquals(targetNumFiles, indexStats.reusedFileCount()); } else { - assertEquals(indexStats.reusedFileCount(), 0); + assertEquals(0, indexStats.reusedFileCount(), 0); } DirectoryReader reader = DirectoryReader.open(target); SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target); diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java index 67c431135a0..70eacaafedb 100644 --- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java +++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.Fields; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.test.ESTestCase; @@ -105,11 +105,11 @@ public class FileInfoTests extends ESTestCase { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject(); - builder.field(Fields.NAME, name); - builder.field(Fields.PHYSICAL_NAME, physicalName); - builder.field(Fields.LENGTH, length); - builder.field(Fields.WRITTEN_BY, Version.LATEST.toString()); - builder.field(Fields.CHECKSUM, "666"); + builder.field(FileInfo.NAME, name); + builder.field(FileInfo.PHYSICAL_NAME, physicalName); + builder.field(FileInfo.LENGTH, length); + builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString()); + builder.field(FileInfo.CHECKSUM, "666"); builder.endObject(); byte[] xContent = builder.bytes().toBytes(); diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 2722fc9d9d3..1f1b758f349 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -455,7 +455,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { assertEquals(1, imc.availableShards().size()); assertTrue(newShard.recoverFromStore()); assertTrue("we should have flushed in IMC at least once but did: " + flushes.get(), flushes.get() >= 1); - newShard.updateRoutingEntry(routing.moveToStarted(), true); + newShard.updateRoutingEntry(routing.moveToStarted()); } finally { newShard.close("simon says", false); } diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index 8d59da7da01..92a411a95de 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -102,13 +102,13 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas newRouting = ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom")); newRouting = ShardRoutingHelper.initialize(newRouting, nodeId); IndexShard shard = index.createShard(newRouting); - shard.updateRoutingEntry(newRouting, true); + shard.updateRoutingEntry(newRouting); final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode)); shard.recoverFromStore(); newRouting = ShardRoutingHelper.moveToStarted(newRouting); - shard.updateRoutingEntry(newRouting, true); + shard.updateRoutingEntry(newRouting); } finally { indicesService.deleteIndex(idx, "simon says"); } diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 5a4aa2e6b24..cd94ee0f8e9 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesQueryCache; @@ -54,12 +55,12 @@ public class IndicesQueryCacheTests extends ESTestCase { @Override public boolean equals(Object obj) { - return super.equals(obj) && id == ((DummyQuery) obj).id; + return sameClassAs(obj) && id == ((DummyQuery) obj).id; } @Override public int hashCode() { - return 31 * super.hashCode() + id; + return 31 * classHash() + id; } @Override @@ -93,6 +94,7 @@ public class IndicesQueryCacheTests extends ESTestCase { Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) .build(); IndicesQueryCache cache = new IndicesQueryCache(settings); s.setQueryCache(cache); @@ -173,6 +175,7 @@ public class IndicesQueryCacheTests extends ESTestCase { Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) .build(); IndicesQueryCache cache = new IndicesQueryCache(settings); s1.setQueryCache(cache); @@ -298,6 +301,7 @@ public class IndicesQueryCacheTests extends ESTestCase { Settings settings = Settings.builder() .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10) + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) .build(); IndicesQueryCache cache = new IndicesQueryCache(settings); s1.setQueryCache(cache); diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java index 646d9651436..1cca3bb7215 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java @@ -30,9 +30,9 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.cache.RemovalNotification; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -58,29 +58,30 @@ public class IndicesRequestCacheTests extends ESTestCase { new ShardId("foo", "bar", 1)); TermQueryBuilder termQuery = new TermQueryBuilder("id", "0"); AtomicBoolean indexShard = new AtomicBoolean(true); - TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); // initial cache + TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); + assertEquals("foo", StreamInput.wrap(value).readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); + assertFalse(entity.loadedFromCache()); assertEquals(1, cache.count()); // cache hit + entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); + assertEquals("foo", StreamInput.wrap(value).readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); + assertTrue(entity.loadedFromCache()); assertEquals(1, cache.count()); assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length()); assertEquals(1, cache.numRegisteredCloseListeners()); - // release + // Closing the cache doesn't modify an already returned CacheEntity if (randomBoolean()) { reader.close(); } else { @@ -91,7 +92,7 @@ public class IndicesRequestCacheTests extends ESTestCase { assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); + assertTrue(entity.loadedFromCache()); assertEquals(0, cache.count()); assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt()); @@ -99,43 +100,6 @@ public class IndicesRequestCacheTests extends ESTestCase { assertEquals(0, cache.numRegisteredCloseListeners()); } - public void testCacheWithDifferentEntityInstance() throws Exception { - IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY); - AtomicBoolean indexShard = new AtomicBoolean(true); - ShardRequestCache requestCacheStats = new ShardRequestCache(); - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); - - writer.addDocument(newDoc(0, "foo")); - DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), - new ShardId("foo", "bar", 1)); - TermQueryBuilder termQuery = new TermQueryBuilder("id", "0"); - TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); - - // initial cache - BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); - assertEquals(0, requestCacheStats.stats().getHitCount()); - assertEquals(1, requestCacheStats.stats().getMissCount()); - assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, cache.count()); - assertEquals(1, cache.numRegisteredCloseListeners()); - final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt(); - - value = cache.getOrCompute(new TestEntity(requestCacheStats, reader, indexShard, 0), reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); - assertEquals(1, requestCacheStats.stats().getHitCount()); - assertEquals(1, requestCacheStats.stats().getMissCount()); - assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, cache.count()); - assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt()); - - assertEquals(1, cache.numRegisteredCloseListeners()); - IOUtils.close(reader, writer, dir, cache); - } - public void testCacheDifferentReaders() throws Exception { IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY); AtomicBoolean indexShard = new AtomicBoolean(true); @@ -146,62 +110,60 @@ public class IndicesRequestCacheTests extends ESTestCase { writer.addDocument(newDoc(0, "foo")); DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); TermQueryBuilder termQuery = new TermQueryBuilder("id", "0"); - TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); writer.updateDocument(new Term("id", "0"), newDoc(0, "bar")); DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); - TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); // initial cache + TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); + assertEquals("foo", StreamInput.wrap(value).readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); + assertFalse(entity.loadedFromCache()); assertEquals(1, cache.count()); assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length()); final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt(); assertEquals(1, cache.numRegisteredCloseListeners()); // cache the second + TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", value.toUtf8()); + assertEquals("bar", StreamInput.wrap(value).readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, secondEntity.loaded); + assertFalse(secondEntity.loadedFromCache()); assertEquals(2, cache.count()); assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > cacheSize + value.length()); assertEquals(2, cache.numRegisteredCloseListeners()); - - + secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", value.toUtf8()); + assertEquals("bar", StreamInput.wrap(value).readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, secondEntity.loaded); + assertTrue(secondEntity.loadedFromCache()); assertEquals(2, cache.count()); + entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value.toUtf8()); + assertEquals("foo", StreamInput.wrap(value).readString()); assertEquals(2, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, secondEntity.loaded); + assertTrue(entity.loadedFromCache()); assertEquals(2, cache.count()); + // Closing the cache doesn't change returned entities reader.close(); cache.cleanCache(); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, secondEntity.loaded); + assertTrue(entity.loadedFromCache()); + assertTrue(secondEntity.loadedFromCache()); assertEquals(1, cache.count()); assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt()); assertEquals(1, cache.numRegisteredCloseListeners()); @@ -217,14 +179,13 @@ public class IndicesRequestCacheTests extends ESTestCase { cache.cleanCache(); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); - assertEquals(1, entity.loaded); - assertEquals(1, secondEntity.loaded); + assertTrue(entity.loadedFromCache()); + assertTrue(secondEntity.loadedFromCache()); assertEquals(0, cache.count()); assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt()); IOUtils.close(secondReader, writer, dir, cache); assertEquals(0, cache.numRegisteredCloseListeners()); - } public void testEviction() throws Exception { @@ -248,9 +209,9 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value1.toUtf8()); + assertEquals("foo", StreamInput.wrap(value1).readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", value2.toUtf8()); + assertEquals("bar", StreamInput.wrap(value2).readString()); size = requestCacheStats.stats().getMemorySize(); IOUtils.close(reader, secondReader, writer, dir, cache); } @@ -279,12 +240,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value1.toUtf8()); + assertEquals("foo", StreamInput.wrap(value1).readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", value2.toUtf8()); + assertEquals("bar", StreamInput.wrap(value2).readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", value3.toUtf8()); + assertEquals("baz", StreamInput.wrap(value3).readString()); assertEquals(2, cache.count()); assertEquals(1, requestCacheStats.stats().getEvictions()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); @@ -316,12 +277,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", value1.toUtf8()); + assertEquals("foo", StreamInput.wrap(value1).readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", value2.toUtf8()); + assertEquals("bar", StreamInput.wrap(value2).readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", value3.toUtf8()); + assertEquals("baz", StreamInput.wrap(value3).readString()); assertEquals(3, cache.count()); final long hitCount = requestCacheStats.stats().getHitCount(); // clear all for the indexShard Idendity even though is't still open @@ -331,7 +292,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // third has not been validated since it's a different identity value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount()); - assertEquals("baz", value3.toUtf8()); + assertEquals("baz", StreamInput.wrap(value3).readString()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); @@ -343,59 +304,37 @@ public class IndicesRequestCacheTests extends ESTestCase { StringField.TYPE_STORED)); } - private class TestEntity implements IndicesRequestCache.CacheEntity { - private final DirectoryReader reader; - private final int id; - private final AtomicBoolean identity; + private class TestEntity extends AbstractIndexShardCacheEntity { + private final AtomicBoolean standInForIndexShard; private final ShardRequestCache shardRequestCache; - private int loaded; - private TestEntity(ShardRequestCache shardRequestCache, DirectoryReader reader, AtomicBoolean identity, int id) { - this.reader = reader; - this.id = id; - this.identity = identity; + private TestEntity(ShardRequestCache shardRequestCache, DirectoryReader reader, AtomicBoolean standInForIndexShard, int id) { + super(new Loader() { + @Override + public void load(StreamOutput out) throws IOException { + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1); + assertEquals(1, topDocs.totalHits); + Document document = reader.document(topDocs.scoreDocs[0].doc); + out.writeString(document.get("value")); + } + }); + this.standInForIndexShard = standInForIndexShard; this.shardRequestCache = shardRequestCache; } @Override - public IndicesRequestCache.Value loadValue() throws IOException { - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(this.id))), 1); - assertEquals(1, topDocs.totalHits); - Document document = reader.document(topDocs.scoreDocs[0].doc); - BytesArray value = new BytesArray(document.get("value")); - loaded++; - return new IndicesRequestCache.Value(value, value.length()); - } - - @Override - public void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) { - shardRequestCache.onCached(key, value); + protected ShardRequestCache stats() { + return shardRequestCache; } @Override public boolean isOpen() { - return identity.get(); + return standInForIndexShard.get(); } @Override public Object getCacheIdentity() { - return identity; - } - - @Override - public void onHit() { - shardRequestCache.onHit(); - } - - @Override - public void onMiss() { - shardRequestCache.onMiss(); - } - - @Override - public void onRemoval(RemovalNotification notification) { - shardRequestCache.onRemoval(notification.getKey(), notification.getValue(), - notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED); + return standInForIndexShard; } } } diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java new file mode 100644 index 00000000000..69bee510710 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -0,0 +1,318 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.cluster; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.Callback; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardState; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndex; +import org.elasticsearch.indices.cluster.IndicesClusterStateService.Shard; +import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.indices.recovery.RecoveryTargetService; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; +import static org.hamcrest.Matchers.equalTo; + +/** + * Abstract base class for tests against {@link IndicesClusterStateService} + */ +public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestCase { + + + protected void failRandomly() { + if (rarely()) { + throw new RuntimeException("dummy test failure"); + } + } + + /** + * Checks if cluster state matches internal state of IndicesClusterStateService instance + * + * @param state cluster state used for matching + */ + public static void assertClusterStateMatchesNodeState(ClusterState state, IndicesClusterStateService indicesClusterStateService) { + AllocatedIndices> indicesService = + indicesClusterStateService.indicesService; + ConcurrentMap failedShardsCache = indicesClusterStateService.failedShardsCache; + RoutingNode localRoutingNode = state.getRoutingNodes().node(state.getNodes().getLocalNodeId()); + if (localRoutingNode != null) { + // check that all shards in local routing nodes have been allocated + for (ShardRouting shardRouting : localRoutingNode) { + Index index = shardRouting.index(); + IndexMetaData indexMetaData = state.metaData().getIndexSafe(index); + + Shard shard = indicesService.getShardOrNull(shardRouting.shardId()); + ShardRouting failedShard = failedShardsCache.get(shardRouting.shardId()); + if (shard == null && failedShard == null) { + fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache"); + } + if (failedShard != null && failedShard.isSameAllocation(shardRouting) == false) { + fail("Shard cache has not been properly cleaned for " + failedShard); + } + + if (shard != null) { + AllocatedIndex indexService = indicesService.indexService(index); + assertTrue("Index " + index + " expected but missing in indicesService", indexService != null); + + // index metadata has been updated + assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData)); + // shard has been created + if (failedShard == null) { + assertTrue("Shard with id " + shardRouting + " expected but missing in indexService", + shard != null); + // shard has latest shard routing + assertThat(shard.routingEntry(), equalTo(shardRouting)); + } + } + } + } + + // all other shards / indices have been cleaned up + for (AllocatedIndex indexService : indicesService) { + assertTrue(state.metaData().getIndexSafe(indexService.index()) != null); + + boolean shardsFound = false; + for (Shard shard : indexService) { + shardsFound = true; + ShardRouting persistedShardRouting = shard.routingEntry(); + boolean found = false; + for (ShardRouting shardRouting : localRoutingNode) { + if (persistedShardRouting.equals(shardRouting)) { + found = true; + } + } + assertTrue(found); + } + + if (shardsFound == false) { + // check if we have shards of that index in failedShardsCache + // if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread + assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index()))); + } + + } + } + + /** + * Mock for {@link IndicesService} + */ + protected class MockIndicesService implements AllocatedIndices { + private volatile Map indices = emptyMap(); + + @Override + public synchronized MockIndexService createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, + List buildInIndexListener) throws IOException { + MockIndexService indexService = new MockIndexService(new IndexSettings(indexMetaData, Settings.EMPTY)); + indices = newMapBuilder(indices).put(indexMetaData.getIndexUUID(), indexService).immutableMap(); + return indexService; + } + + @Override + public IndexMetaData verifyIndexIsDeleted(Index index, ClusterState state) { + return null; + } + + @Override + public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { + + } + + @Override + public synchronized void deleteIndex(Index index, String reason) { + if (hasIndex(index) == false) { + return; + } + Map newIndices = new HashMap<>(indices); + newIndices.remove(index.getUUID()); + indices = unmodifiableMap(newIndices); + } + + @Override + public synchronized void removeIndex(Index index, String reason) { + if (hasIndex(index) == false) { + return; + } + Map newIndices = new HashMap<>(indices); + newIndices.remove(index.getUUID()); + indices = unmodifiableMap(newIndices); + } + + @Override + public @Nullable MockIndexService indexService(Index index) { + return indices.get(index.getUUID()); + } + + @Override + public MockIndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, + RecoveryTargetService recoveryTargetService, + RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, + NodeServicesProvider nodeServicesProvider, Callback onShardFailure) + throws IOException { + failRandomly(); + MockIndexService indexService = indexService(recoveryState.getShardId().getIndex()); + MockIndexShard indexShard = indexService.createShard(shardRouting); + indexShard.recoveryState = recoveryState; + return indexShard; + } + + @Override + public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeValue) throws IOException, + InterruptedException { + + } + + private boolean hasIndex(Index index) { + return indices.containsKey(index.getUUID()); + } + + @Override + public Iterator iterator() { + return indices.values().iterator(); + } + } + + /** + * Mock for {@link IndexService} + */ + protected class MockIndexService implements AllocatedIndex { + private volatile Map shards = emptyMap(); + + private final IndexSettings indexSettings; + + public MockIndexService(IndexSettings indexSettings) { + this.indexSettings = indexSettings; + } + + @Override + public IndexSettings getIndexSettings() { + return indexSettings; + } + + @Override + public boolean updateMapping(IndexMetaData indexMetaData) throws IOException { + failRandomly(); + return false; + } + + @Override + public void updateMetaData(IndexMetaData indexMetaData) { + indexSettings.updateIndexMetaData(indexMetaData); + } + + @Override + public MockIndexShard getShardOrNull(int shardId) { + return shards.get(shardId); + } + + public synchronized MockIndexShard createShard(ShardRouting routing) throws IOException { + failRandomly(); + MockIndexShard shard = new MockIndexShard(routing); + shards = newMapBuilder(shards).put(routing.id(), shard).immutableMap(); + return shard; + } + + @Override + public synchronized void removeShard(int shardId, String reason) { + if (shards.containsKey(shardId) == false) { + return; + } + HashMap newShards = new HashMap<>(shards); + MockIndexShard indexShard = newShards.remove(shardId); + assert indexShard != null; + shards = unmodifiableMap(newShards); + } + + @Override + public Iterator iterator() { + return shards.values().iterator(); + } + + @Override + public Index index() { + return indexSettings.getIndex(); + } + } + + /** + * Mock for {@link IndexShard} + */ + protected class MockIndexShard implements IndicesClusterStateService.Shard { + private volatile ShardRouting shardRouting; + private volatile RecoveryState recoveryState; + + public MockIndexShard(ShardRouting shardRouting) { + this.shardRouting = shardRouting; + } + + @Override + public ShardId shardId() { + return shardRouting.shardId(); + } + + @Override + public RecoveryState recoveryState() { + return recoveryState; + } + + @Override + public ShardRouting routingEntry() { + return shardRouting; + } + + @Override + public IndexShardState state() { + return null; + } + + @Override + public void updateRoutingEntry(ShardRouting shardRouting) throws IOException { + failRandomly(); + assert this.shardId().equals(shardRouting.shardId()); + assert this.shardRouting.isSameAllocation(shardRouting); + this.shardRouting = shardRouting; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java new file mode 100644 index 00000000000..84e83db6d1d --- /dev/null +++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -0,0 +1,234 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.cluster; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.DestructiveOperations; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.action.support.master.TransportMasterNodeActionUtils; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.metadata.AliasValidator; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService; +import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; +import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; +import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; +import org.elasticsearch.cluster.routing.allocation.RandomAllocationDeciderTests; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.gateway.NoopGatewayAllocator; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; +import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyList; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ClusterStateChanges { + + private final ClusterService clusterService; + private final AllocationService allocationService; + + // transport actions + private final TransportCloseIndexAction transportCloseIndexAction; + private final TransportOpenIndexAction transportOpenIndexAction; + private final TransportDeleteIndexAction transportDeleteIndexAction; + private final TransportUpdateSettingsAction transportUpdateSettingsAction; + private final TransportClusterRerouteAction transportClusterRerouteAction; + private final TransportCreateIndexAction transportCreateIndexAction; + + public ClusterStateChanges() { + Settings settings = Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build(); + + allocationService = new AllocationService(settings, new AllocationDeciders(settings, + new HashSet<>(Arrays.asList(new SameShardAllocationDecider(settings), + new ReplicaAfterPrimaryActiveAllocationDecider(settings), + new RandomAllocationDeciderTests.RandomAllocationDecider(getRandom())))), + NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), + EmptyClusterInfoService.INSTANCE); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + ActionFilters actionFilters = new ActionFilters(Collections.emptySet()); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings); + DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterSettings); + Environment environment = new Environment(settings); + ThreadPool threadPool = null; // it's not used + Transport transport = null; // it's not used + + // mocks + clusterService = mock(ClusterService.class); + IndicesService indicesService = mock(IndicesService.class); + // MetaDataCreateIndexService creates indices using its IndicesService instance to check mappings -> fake it here + try { + when(indicesService.createIndex(any(NodeServicesProvider.class), any(IndexMetaData.class), anyList())) + .then(invocationOnMock -> { + IndexService indexService = mock(IndexService.class); + IndexMetaData indexMetaData = (IndexMetaData)invocationOnMock.getArguments()[1]; + when(indexService.index()).thenReturn(indexMetaData.getIndex()); + MapperService mapperService = mock(MapperService.class); + when(indexService.mapperService()).thenReturn(mapperService); + when(mapperService.docMappers(anyBoolean())).thenReturn(Collections.emptyList()); + when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() {}); + return indexService; + }); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + // services + TransportService transportService = new TransportService(settings, transport, threadPool, null); + MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, null, null) { + // metaData upgrader should do nothing + @Override + public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData) { + return indexMetaData; + } + }; + NodeServicesProvider nodeServicesProvider = new NodeServicesProvider(threadPool, null, null, null, null, null, clusterService); + MetaDataIndexStateService indexStateService = new MetaDataIndexStateService(settings, clusterService, allocationService, + metaDataIndexUpgradeService, nodeServicesProvider, indicesService); + MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService); + MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService, + allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new IndexNameExpressionResolver(settings)); + MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService, + allocationService, Version.CURRENT, new AliasValidator(settings), Collections.emptySet(), environment, + nodeServicesProvider, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); + + transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool, + indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations); + transportOpenIndexAction = new TransportOpenIndexAction(settings, transportService, + clusterService, threadPool, indexStateService, actionFilters, indexNameExpressionResolver, destructiveOperations); + transportDeleteIndexAction = new TransportDeleteIndexAction(settings, transportService, + clusterService, threadPool, deleteIndexService, actionFilters, indexNameExpressionResolver, destructiveOperations); + transportUpdateSettingsAction = new TransportUpdateSettingsAction(settings, + transportService, clusterService, threadPool, metaDataUpdateSettingsService, actionFilters, indexNameExpressionResolver); + transportClusterRerouteAction = new TransportClusterRerouteAction(settings, + transportService, clusterService, threadPool, allocationService, actionFilters, indexNameExpressionResolver); + transportCreateIndexAction = new TransportCreateIndexAction(settings, + transportService, clusterService, threadPool, createIndexService, actionFilters, indexNameExpressionResolver); + } + + public ClusterState createIndex(ClusterState state, CreateIndexRequest request) { + return execute(transportCreateIndexAction, request, state); + } + + public ClusterState closeIndices(ClusterState state, CloseIndexRequest request) { + return execute(transportCloseIndexAction, request, state); + } + + public ClusterState openIndices(ClusterState state, OpenIndexRequest request) { + return execute(transportOpenIndexAction, request, state); + } + + public ClusterState deleteIndices(ClusterState state, DeleteIndexRequest request) { + return execute(transportDeleteIndexAction, request, state); + } + + public ClusterState updateSettings(ClusterState state, UpdateSettingsRequest request) { + return execute(transportUpdateSettingsAction, request, state); + } + + public ClusterState reroute(ClusterState state, ClusterRerouteRequest request) { + return execute(transportClusterRerouteAction, request, state); + } + + public ClusterState applyFailedShards(ClusterState clusterState, List failedShards) { + RoutingAllocation.Result rerouteResult = allocationService.applyFailedShards(clusterState, failedShards); + return ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + } + + public ClusterState applyStartedShards(ClusterState clusterState, List startedShards) { + RoutingAllocation.Result rerouteResult = allocationService.applyStartedShards(clusterState, startedShards); + return ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + } + + private , Response extends ActionResponse> ClusterState execute( + TransportMasterNodeAction masterNodeAction, Request request, ClusterState clusterState) { + return executeClusterStateUpdateTask(clusterState, () -> { + try { + TransportMasterNodeActionUtils.runMasterOperation(masterNodeAction, request, clusterState, new PlainActionFuture<>()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private ClusterState executeClusterStateUpdateTask(ClusterState state, Runnable runnable) { + ClusterState[] result = new ClusterState[1]; + doAnswer(invocationOnMock -> { + ClusterStateUpdateTask task = (ClusterStateUpdateTask)invocationOnMock.getArguments()[1]; + result[0] = task.execute(state); + return null; + }).when(clusterService).submitStateUpdateTask(anyString(), any(ClusterStateUpdateTask.class)); + runnable.run(); + assertThat(result[0], notNullValue()); + return result[0]; + } +} diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java new file mode 100644 index 00000000000..8c63c001a1e --- /dev/null +++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -0,0 +1,281 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.cluster; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.FailedShard; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.DummyTransportAddress; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.indices.recovery.RecoveryTargetService; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Executor; + +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndicesClusterStateServiceTestCase { + + private final ClusterStateChanges cluster = new ClusterStateChanges(); + + public void testRandomClusterStateUpdates() { + // we have an IndicesClusterStateService per node in the cluster + final Map clusterStateServiceMap = new HashMap<>(); + ClusterState state = randomInitialClusterState(clusterStateServiceMap); + + // each of the following iterations represents a new cluster state update processed on all nodes + for (int i = 0; i < 30; i++) { + logger.info("Iteration {}", i); + final ClusterState previousState = state; + + // calculate new cluster state + for (int j = 0; j < randomInt(3); j++) { // multiple iterations to simulate batching of cluster states + state = randomlyUpdateClusterState(state, clusterStateServiceMap); + } + + // apply cluster state to nodes (incl. master) + for (DiscoveryNode node : state.nodes()) { + IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node); + ClusterState localState = adaptClusterStateToLocalNode(state, node); + ClusterState previousLocalState = adaptClusterStateToLocalNode(previousState, node); + indicesClusterStateService.clusterChanged(new ClusterChangedEvent("simulated change " + i, localState, previousLocalState)); + + // check that cluster state has been properly applied to node + assertClusterStateMatchesNodeState(localState, indicesClusterStateService); + } + } + + // TODO: check if we can go to green by starting all shards and finishing all iterations + logger.info("Final cluster state: {}", state.prettyPrint()); + } + + public ClusterState randomInitialClusterState(Map clusterStateServiceMap) { + List allNodes = new ArrayList<>(); + DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master + allNodes.add(localNode); + // at least two nodes that have the data role so that we can allocate shards + allNodes.add(createNode(DiscoveryNode.Role.DATA)); + allNodes.add(createNode(DiscoveryNode.Role.DATA)); + for (int i = 0; i < randomIntBetween(2, 5); i++) { + allNodes.add(createNode()); + } + ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[allNodes.size()])); + // add nodes to clusterStateServiceMap + updateNodes(state, clusterStateServiceMap); + return state; + } + + private void updateNodes(ClusterState state, Map clusterStateServiceMap) { + for (DiscoveryNode node : state.nodes()) { + clusterStateServiceMap.computeIfAbsent(node, discoveryNode -> { + IndicesClusterStateService ics = createIndicesClusterStateService(); + ics.start(); + return ics; + }); + } + + for (Iterator> it = clusterStateServiceMap.entrySet().iterator(); it.hasNext(); ) { + DiscoveryNode node = it.next().getKey(); + if (state.nodes().nodeExists(node.getId()) == false) { + it.remove(); + } + } + } + + public ClusterState randomlyUpdateClusterState(ClusterState state, + Map clusterStateServiceMap) { + // randomly create new indices (until we have 200 max) + for (int i = 0; i < randomInt(5); i++) { + if (state.metaData().indices().size() > 200) { + break; + } + String name = "index_" + randomAsciiOfLength(15).toLowerCase(Locale.ROOT); + CreateIndexRequest request = new CreateIndexRequest(name, Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3)) + .put(SETTING_NUMBER_OF_REPLICAS, randomInt(2)) + .build()); + state = cluster.createIndex(state, request); + assertTrue(state.metaData().hasIndex(name)); + } + + // randomly delete indices + Set indicesToDelete = new HashSet<>(); + int numberOfIndicesToDelete = randomInt(Math.min(2, state.metaData().indices().size())); + for (String index : randomSubsetOf(numberOfIndicesToDelete, state.metaData().indices().keys().toArray(String.class))) { + indicesToDelete.add(state.metaData().index(index).getIndex().getName()); + } + if (indicesToDelete.isEmpty() == false) { + DeleteIndexRequest deleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[indicesToDelete.size()])); + state = cluster.deleteIndices(state, deleteRequest); + for (String index : indicesToDelete) { + assertFalse(state.metaData().hasIndex(index)); + } + } + + // randomly close indices + int numberOfIndicesToClose = randomInt(Math.min(1, state.metaData().indices().size())); + for (String index : randomSubsetOf(numberOfIndicesToClose, state.metaData().indices().keys().toArray(String.class))) { + CloseIndexRequest closeIndexRequest = new CloseIndexRequest(state.metaData().index(index).getIndex().getName()); + state = cluster.closeIndices(state, closeIndexRequest); + } + + // randomly open indices + int numberOfIndicesToOpen = randomInt(Math.min(1, state.metaData().indices().size())); + for (String index : randomSubsetOf(numberOfIndicesToOpen, state.metaData().indices().keys().toArray(String.class))) { + OpenIndexRequest openIndexRequest = new OpenIndexRequest(state.metaData().index(index).getIndex().getName()); + state = cluster.openIndices(state, openIndexRequest); + } + + // randomly update settings + Set indicesToUpdate = new HashSet<>(); + boolean containsClosedIndex = false; + int numberOfIndicesToUpdate = randomInt(Math.min(2, state.metaData().indices().size())); + for (String index : randomSubsetOf(numberOfIndicesToUpdate, state.metaData().indices().keys().toArray(String.class))) { + indicesToUpdate.add(state.metaData().index(index).getIndex().getName()); + if (state.metaData().index(index).getState() == IndexMetaData.State.CLOSE) { + containsClosedIndex = true; + } + } + if (indicesToUpdate.isEmpty() == false) { + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest( + indicesToUpdate.toArray(new String[indicesToUpdate.size()])); + Settings.Builder settings = Settings.builder(); + if (containsClosedIndex == false) { + settings.put(SETTING_NUMBER_OF_REPLICAS, randomInt(2)); + } + settings.put("index.refresh_interval", randomIntBetween(1, 5) + "s"); + updateSettingsRequest.settings(settings.build()); + state = cluster.updateSettings(state, updateSettingsRequest); + } + + // randomly reroute + if (rarely()) { + state = cluster.reroute(state, new ClusterRerouteRequest()); + } + + // randomly start and fail allocated shards + List startedShards = new ArrayList<>(); + List failedShards = new ArrayList<>(); + for (DiscoveryNode node : state.nodes()) { + IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node); + MockIndicesService indicesService = (MockIndicesService) indicesClusterStateService.indicesService; + for (MockIndexService indexService : indicesService) { + for (MockIndexShard indexShard : indexService) { + ShardRouting persistedShardRouting = indexShard.routingEntry(); + if (persistedShardRouting.initializing() && randomBoolean()) { + startedShards.add(persistedShardRouting); + } else if (rarely()) { + failedShards.add(new FailedShard(persistedShardRouting, "fake shard failure", new Exception())); + } + } + } + } + state = cluster.applyFailedShards(state, failedShards); + state = cluster.applyStartedShards(state, startedShards); + + // randomly add and remove nodes (except current master) + if (rarely()) { + if (randomBoolean()) { + // add node + if (state.nodes().getSize() < 10) { + DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).put(createNode()).build(); + state = ClusterState.builder(state).nodes(newNodes).build(); + state = cluster.reroute(state, new ClusterRerouteRequest()); // always reroute after node leave + updateNodes(state, clusterStateServiceMap); + } + } else { + // remove node + if (state.nodes().getDataNodes().size() > 3) { + DiscoveryNode discoveryNode = randomFrom(state.nodes().getNodes().values().toArray(DiscoveryNode.class)); + if (discoveryNode.equals(state.nodes().getMasterNode()) == false) { + DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).remove(discoveryNode.getId()).build(); + state = ClusterState.builder(state).nodes(newNodes).build(); + state = cluster.reroute(state, new ClusterRerouteRequest()); // always reroute after node join + updateNodes(state, clusterStateServiceMap); + } + } + } + } + + // TODO: go masterless? + + return state; + } + + protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) { + Set roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values()))); + for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) { + roles.add(mustHaveRole); + } + return new DiscoveryNode("node_" + randomAsciiOfLength(8), DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles, + Version.CURRENT); + } + + private static ClusterState adaptClusterStateToLocalNode(ClusterState state, DiscoveryNode node) { + return ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(node.getId())).build(); + } + + private IndicesClusterStateService createIndicesClusterStateService() { + final ThreadPool threadPool = mock(ThreadPool.class); + final Executor executor = mock(Executor.class); + when(threadPool.generic()).thenReturn(executor); + final MockIndicesService indicesService = new MockIndicesService(); + final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool, null); + final ClusterService clusterService = mock(ClusterService.class); + final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService, + transportService, null, null); + final RecoveryTargetService recoveryTargetService = new RecoveryTargetService(Settings.EMPTY, threadPool, + transportService, null, clusterService); + final ShardStateAction shardStateAction = mock(ShardStateAction.class); + return new IndicesClusterStateService(Settings.EMPTY, indicesService, clusterService, + threadPool, recoveryTargetService, shardStateAction, null, repositoriesService, null, null, null, null, null); + } + +} diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 3565cf0147d..a4096fde9da 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.search.sort.SortOrder; @@ -78,6 +79,7 @@ public class IndexStatsIT extends ESIntegTestCase { //Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad return Settings.builder().put(super.nodeSettings(nodeOrdinal)) .put(IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), "1ms") + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) .build(); } diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 101d2d8e50d..9cade7aa513 100644 --- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -91,9 +91,39 @@ public class RestControllerTests extends ESTestCase { restHeaders.put("header.1", "true"); restHeaders.put("header.2", "true"); restHeaders.put("header.3", "false"); - restController.dispatchRequest(new FakeRestRequest(restHeaders), null, threadContext); + restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, threadContext); assertNull(threadContext.getHeader("header.1")); assertNull(threadContext.getHeader("header.2")); assertEquals("true", threadContext.getHeader("header.3")); } + + public void testCanTripCircuitBreaker() throws Exception { + RestController controller = new RestController(Settings.EMPTY); + // trip circuit breaker by default + controller.registerHandler(RestRequest.Method.GET, "/trip", new FakeRestHandler(true)); + controller.registerHandler(RestRequest.Method.GET, "/do-not-trip", new FakeRestHandler(false)); + + assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/trip").build())); + // assume trip even on unknown paths + assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/unknown-path").build())); + assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/do-not-trip").build())); + } + + private static class FakeRestHandler implements RestHandler { + private final boolean canTripCircuitBreaker; + + private FakeRestHandler(boolean canTripCircuitBreaker) { + this.canTripCircuitBreaker = canTripCircuitBreaker; + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel) throws Exception { + //no op + } + + @Override + public boolean canTripCircuitBreaker() { + return canTripCircuitBreaker; + } + } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java index bb0497ed343..ebb7dd255aa 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -82,7 +81,7 @@ public class RestMainActionTests extends ESTestCase { if (prettyPrint == false) { params.put("pretty", String.valueOf(prettyPrint)); } - RestRequest restRequest = new FakeRestRequest(Collections.emptyMap(), params); + RestRequest restRequest = new FakeRestRequest.Builder().withParams(params).build(); BytesRestResponse response = RestMainAction.convertMainResponse(mainResponse, restRequest, builder); assertNotNull(response); diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java index 89b3b6fd239..a7e17785d48 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java @@ -146,7 +146,7 @@ public class RestTableTests extends ESTestCase { } private RestResponse assertResponseContentType(Map headers, String mediaType) throws Exception { - FakeRestRequest requestWithAcceptHeader = new FakeRestRequest(headers); + FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder().withHeaders(headers).build(); table.startRow(); table.addCell("foo"); table.addCell("foo"); diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 384e181c899..72873bc0d48 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -29,8 +29,9 @@ import org.elasticsearch.index.query.QueryParser; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.search.highlight.CustomHighlighter; -import org.elasticsearch.search.highlight.Highlighter; +import org.elasticsearch.search.highlight.FastVectorHighlighter; import org.elasticsearch.search.highlight.PlainHighlighter; +import org.elasticsearch.search.highlight.PostingsHighlighter; import org.elasticsearch.search.suggest.CustomSuggester; import org.elasticsearch.search.suggest.phrase.PhraseSuggester; @@ -48,7 +49,7 @@ public class SearchModuleTests extends ModuleTestCase { public void testDoubleRegister() { SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry()); try { - module.registerHighlighter("fvh", PlainHighlighter.class); + module.registerHighlighter("fvh", new PlainHighlighter()); } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Can't register the same [highlighter] more than once for [fvh]"); } @@ -70,13 +71,20 @@ public class SearchModuleTests extends ModuleTestCase { public void testRegisterHighlighter() { SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry()); - module.registerHighlighter("custom", CustomHighlighter.class); - try { - module.registerHighlighter("custom", CustomHighlighter.class); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), "Can't register the same [highlighter] more than once for [custom]"); - } - assertMapMultiBinding(module, Highlighter.class, CustomHighlighter.class); + CustomHighlighter customHighlighter = new CustomHighlighter(); + module.registerHighlighter("custom", customHighlighter); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> module.registerHighlighter("custom", new CustomHighlighter())); + assertEquals("Can't register the same [highlighter] more than once for [custom]", exception.getMessage()); + + exception = expectThrows(IllegalArgumentException.class, + () -> module.registerHighlighter("custom", null)); + assertEquals("Can't register null highlighter for key: [custom]", exception.getMessage()); + Highlighters highlighters = module.getHighlighters(); + assertEquals(highlighters.get("fvh").getClass(), FastVectorHighlighter.class); + assertEquals(highlighters.get("plain").getClass(), PlainHighlighter.class); + assertEquals(highlighters.get("postings").getClass(), PostingsHighlighter.class); + assertSame(highlighters.get("custom"), customHighlighter); } public void testRegisterQueryParserDuplicate() { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java index aad2c9bb3ed..e38f33cfa2b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java @@ -156,7 +156,7 @@ public class BooleanTermsIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) - .size(randomInt(5)) + .size(between(1, 5)) .collectMode(randomFrom(SubAggCollectionMode.values()))) .execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java index 6f2d7c8fb93..b22f7524e32 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java @@ -134,10 +134,9 @@ public class ChildrenIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("test") .setQuery(matchQuery("randomized", true)) .addAggregation( - terms("category").field("category").size(0).subAggregation( -children("to_comment", "comment") + terms("category").field("category").size(10000).subAggregation(children("to_comment", "comment") .subAggregation( - terms("commenters").field("commenter").size(0).subAggregation( + terms("commenters").field("commenter").size(10000).subAggregation( topHits("top_comments") )) ) @@ -176,7 +175,7 @@ children("to_comment", "comment") SearchResponse searchResponse = client().prepareSearch("test") .setQuery(matchQuery("randomized", false)) .addAggregation( - terms("category").field("category").size(0).subAggregation( + terms("category").field("category").size(10000).subAggregation( children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_uid", SortOrder.ASC)) ) ).get(); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 269d22914c1..711d3373590 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; + import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -52,8 +53,8 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; @ESIntegTestCase.SuiteScopeTestCase public class GeoHashGridIT extends ESIntegTestCase { @@ -305,24 +306,24 @@ public class GeoHashGridIT extends ESIntegTestCase { } } - // making sure this doesn't runs into an OOME public void testSizeIsZero() { - for (int precision = 1; precision <= PRECISION; precision++) { - final int size = randomBoolean() ? 0 : randomIntBetween(1, Integer.MAX_VALUE); - final int shardSize = randomBoolean() ? -1 : 0; - SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid") - .field("location") - .size(size) - .shardSize(shardSize) - .precision(precision) - ) - .execute().actionGet(); + final int size = 0; + final int shardSize = 10000; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client().prepareSearch("idx") + .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)).execute() + .actionGet()); + assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geohashgrid]")); + } - assertSearchResponse(response); - GeoHashGrid geoGrid = response.getAggregations().get("geohashgrid"); - assertThat(geoGrid.getBuckets().size(), greaterThanOrEqualTo(1)); - } + public void testShardSizeIsZero() { + final int size = 100; + final int shardSize = 0; + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client().prepareSearch("idx") + .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .execute().actionGet()); + assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geohashgrid]")); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java index c3c8f6902b3..688289f02dd 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java @@ -33,33 +33,10 @@ public class GeoHashGridTests extends BaseAggregationTestCase factory.missing("MISSING"); } if (randomBoolean()) { - int size = randomInt(4); - switch (size) { - case 0: - break; - case 1: - case 2: - case 3: - case 4: - size = randomInt(); - break; - default: - fail(); - } - factory.bucketCountThresholds().setRequiredSize(size); - + factory.bucketCountThresholds().setRequiredSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - int shardSize = randomInt(4); - switch (shardSize) { - case 0: - break; - case 1: - case 2: - case 3: - case 4: - shardSize = randomInt(); - break; - default: - fail(); - } - factory.bucketCountThresholds().setShardSize(shardSize); + factory.bucketCountThresholds().setShardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { int minDocCount = randomInt(4); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index cf8830aa7ef..81c923ccf6d 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -112,11 +112,11 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { } public void onModule(SearchModule searchModule) { - searchModule.registerFetchSubPhase(TermVectorsFetchSubPhase.class); + searchModule.registerFetchSubPhase(new TermVectorsFetchSubPhase()); } } - public static class TermVectorsFetchSubPhase implements FetchSubPhase { + public final static class TermVectorsFetchSubPhase implements FetchSubPhase { public static final ContextFactory CONTEXT_FACTORY = new ContextFactory() { @@ -138,22 +138,11 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase { return singletonMap("term_vectors_fetch", new TermVectorsFetchParseElement()); } - @Override - public boolean hitsExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded(); - } - @Override public void hitExecute(SearchContext context, HitContext hitContext) { + if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) { + return; + } String field = context.getFetchSubPhaseContext(CONTEXT_FACTORY).getField(); if (hitContext.hit().fieldsOrNull() == null) { diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java index 7e01f575822..398ef64bc92 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java @@ -126,6 +126,8 @@ public class GeoBoundingBoxIT extends ESIntegTestCase { } } + // norelease + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-7325") public void testLimitsBoundingBox() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index b8b04a8bc33..0debdb263af 100644 --- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -30,7 +30,6 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.query.SpatialArgs; import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.spatial.query.UnsupportedSpatialOperation; -import org.apache.lucene.spatial.util.GeoProjectionUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -42,6 +41,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.builders.CoordinatesBuilder; import org.elasticsearch.common.geo.builders.LineStringBuilder; import org.elasticsearch.common.geo.builders.MultiPolygonBuilder; @@ -540,7 +540,7 @@ public class GeoFilterIT extends ESIntegTestCase { } public static double distance(double lat1, double lon1, double lat2, double lon2) { - return GeoProjectionUtils.SEMIMAJOR_AXIS * DistanceUtils.distHaversineRAD( + return GeoUtils.EARTH_SEMI_MAJOR_AXIS * DistanceUtils.distHaversineRAD( DistanceUtils.toRadians(lat1), DistanceUtils.toRadians(lon1), DistanceUtils.toRadians(lat2), diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java index 80e39a2a6dd..b6c42b73677 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java @@ -35,6 +35,6 @@ public class CustomHighlighterPlugin extends Plugin { } public void onModule(SearchModule highlightModule) { - highlightModule.registerHighlighter("test-custom", CustomHighlighter.class); + highlightModule.registerHighlighter("test-custom", new CustomHighlighter()); } } diff --git a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java similarity index 70% rename from core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java rename to core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java index d3ee811be23..ccc3102f3a8 100644 --- a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.msearch; +import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -29,9 +30,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.hamcrest.Matchers.equalTo; -/** - */ -public class SimpleMultiSearchIT extends ESIntegTestCase { +public class MultiSearchIT extends ESIntegTestCase { + public void testSimpleMultiSearch() { createIndex("test"); ensureGreen(); @@ -54,4 +54,30 @@ public class SimpleMultiSearchIT extends ESIntegTestCase { assertFirstHit(response.getResponses()[0].getResponse(), hasId("1")); assertFirstHit(response.getResponses()[1].getResponse(), hasId("2")); } + + public void testSimpleMultiSearchMoreRequests() { + createIndex("test"); + int numDocs = randomIntBetween(0, 16); + for (int i = 0; i < numDocs; i++) { + client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get(); + } + refresh(); + + int numSearchRequests = randomIntBetween(1, 64); + MultiSearchRequest request = new MultiSearchRequest(); + if (randomBoolean()) { + request.maxConcurrentSearchRequests(randomIntBetween(1, numSearchRequests)); + } + for (int i = 0; i < numSearchRequests; i++) { + request.add(client().prepareSearch("test")); + } + + MultiSearchResponse response = client().multiSearch(request).actionGet(); + assertThat(response.getResponses().length, equalTo(numSearchRequests)); + for (MultiSearchResponse.Item item : response) { + assertNoFailures(item.getResponse()); + assertHitCount(item.getResponse(), numDocs); + } + } + } diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java index ba378a3c404..e0aec941487 100644 --- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java +++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java @@ -318,13 +318,7 @@ public class SimpleNestedIT extends ESIntegTestCase { assertThat(searchResponse.getHits().totalHits(), equalTo(1L)); Explanation explanation = searchResponse.getHits().hits()[0].explanation(); assertThat(explanation.getValue(), equalTo(2f)); - assertThat(explanation.toString(), startsWith("2.0 = sum of:\n 2.0 = Score based on child doc range from 0 to 1\n")); - // TODO: Enable when changes from BlockJoinQuery#explain are added to Lucene (Most likely version 4.2) -// assertThat(explanation.getDetails().length, equalTo(2)); -// assertThat(explanation.getDetails()[0].getValue(), equalTo(1f)); -// assertThat(explanation.getDetails()[0].getDescription(), equalTo("Child[0]")); -// assertThat(explanation.getDetails()[1].getValue(), equalTo(1f)); -// assertThat(explanation.getDetails()[1].getDescription(), equalTo("Child[1]")); + assertThat(explanation.toString(), startsWith("2.0 = sum of:\n 2.0 = Score based on 2 child docs in range from 0 to 1")); } public void testSimpleNestedSorting() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 8bfadeb9443..49142988a46 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -56,7 +56,7 @@ public class SearchPreferenceIT extends ESIntegTestCase { refresh(); internalCluster().stopRandomDataNode(); client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).execute().actionGet(); - String[] preferences = new String[] {"_primary", "_local", "_primary_first", "_prefer_node:somenode", "_prefer_node:server2"}; + String[] preferences = new String[] {"_primary", "_local", "_primary_first", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"}; for (String pref : preferences) { logger.info("--> Testing out preference={}", pref); SearchResponse searchResponse = client().prepareSearch().setSize(0).setPreference(pref).execute().actionGet(); diff --git a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java new file mode 100644 index 00000000000..848b230b3fa --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java @@ -0,0 +1,403 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.profile.aggregation; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregator; +import org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator; +import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregator; +import org.elasticsearch.search.aggregations.metrics.max.MaxAggregator; +import org.elasticsearch.search.profile.ProfileResult; +import org.elasticsearch.search.profile.ProfileShardResult; +import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult; +import org.elasticsearch.search.profile.aggregation.AggregationTimingType; +import org.elasticsearch.test.ESIntegTestCase; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; +import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.max; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; + +@ESIntegTestCase.SuiteScopeTestCase +public class AggregationProfilerIT extends ESIntegTestCase { + + + private static final String NUMBER_FIELD = "number"; + private static final String TAG_FIELD = "tag"; + private static final String STRING_FIELD = "string_field"; + + @Override + protected int numberOfShards() { + return 1; + } + + @Override + protected void setupSuiteScopeCluster() throws Exception { + assertAcked(client().admin().indices().prepareCreate("idx") + .addMapping("type", STRING_FIELD, "type=keyword", NUMBER_FIELD, "type=integer", TAG_FIELD, "type=keyword").get()); + List builders = new ArrayList<>(); + + String[] randomStrings = new String[randomIntBetween(2, 10)]; + for (int i = 0; i < randomStrings.length; i++) { + randomStrings[i] = randomAsciiOfLength(10); + } + + for (int i = 0; i < 5; i++) { + builders.add(client().prepareIndex("idx", "type").setSource( + jsonBuilder().startObject() + .field(STRING_FIELD, randomFrom(randomStrings)) + .field(NUMBER_FIELD, randomIntBetween(0, 9)) + .field(TAG_FIELD, randomBoolean() ? "more" : "less") + .endObject())); + } + + indexRandom(true, builders); + createIndex("idx_unmapped"); + ensureSearchable(); + } + + public void testSimpleProfile() { + SearchResponse response = client().prepareSearch("idx").setProfile(true) + .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L)).get(); + assertSearchResponse(response); + Map profileResults = response.getProfileResults(); + assertThat(profileResults, notNullValue()); + assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries)); + for (ProfileShardResult profileShardResult : profileResults.values()) { + assertThat(profileShardResult, notNullValue()); + AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults(); + assertThat(aggProfileResults, notNullValue()); + List aggProfileResultsList = aggProfileResults.getProfileResults(); + assertThat(aggProfileResultsList, notNullValue()); + assertThat(aggProfileResultsList.size(), equalTo(1)); + ProfileResult histoAggResult = aggProfileResultsList.get(0); + assertThat(histoAggResult, notNullValue()); + assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName())); + assertThat(histoAggResult.getLuceneDescription(), equalTo("histo")); + assertThat(histoAggResult.getProfiledChildren().size(), equalTo(0)); + assertThat(histoAggResult.getTime(), greaterThan(0L)); + Map breakdown = histoAggResult.getTimeBreakdown(); + assertThat(breakdown, notNullValue()); + assertThat(breakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(breakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(breakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(breakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(breakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(breakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(breakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(breakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + + } + } + + public void testMultiLevelProfile() { + SearchResponse response = client().prepareSearch("idx").setProfile(true) + .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L) + .subAggregation(terms("terms").field(TAG_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)))).get(); + assertSearchResponse(response); + Map profileResults = response.getProfileResults(); + assertThat(profileResults, notNullValue()); + assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries)); + for (ProfileShardResult profileShardResult : profileResults.values()) { + assertThat(profileShardResult, notNullValue()); + AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults(); + assertThat(aggProfileResults, notNullValue()); + List aggProfileResultsList = aggProfileResults.getProfileResults(); + assertThat(aggProfileResultsList, notNullValue()); + assertThat(aggProfileResultsList.size(), equalTo(1)); + ProfileResult histoAggResult = aggProfileResultsList.get(0); + assertThat(histoAggResult, notNullValue()); + assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName())); + assertThat(histoAggResult.getLuceneDescription(), equalTo("histo")); + assertThat(histoAggResult.getTime(), greaterThan(0L)); + Map histoBreakdown = histoAggResult.getTimeBreakdown(); + assertThat(histoBreakdown, notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(histoAggResult.getProfiledChildren().size(), equalTo(1)); + + ProfileResult termsAggResult = histoAggResult.getProfiledChildren().get(0); + assertThat(termsAggResult, notNullValue()); + assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName())); + assertThat(termsAggResult.getLuceneDescription(), equalTo("terms")); + assertThat(termsAggResult.getTime(), greaterThan(0L)); + Map termsBreakdown = termsAggResult.getTimeBreakdown(); + assertThat(termsBreakdown, notNullValue()); + assertThat(termsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(termsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(termsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(termsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(termsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(termsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(termsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(termsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(termsAggResult.getProfiledChildren().size(), equalTo(1)); + + ProfileResult avgAggResult = termsAggResult.getProfiledChildren().get(0); + assertThat(avgAggResult, notNullValue()); + assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName())); + assertThat(avgAggResult.getLuceneDescription(), equalTo("avg")); + assertThat(avgAggResult.getTime(), greaterThan(0L)); + Map avgBreakdown = termsAggResult.getTimeBreakdown(); + assertThat(avgBreakdown, notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0)); + } + } + + public void testComplexProfile() { + SearchResponse response = client().prepareSearch("idx").setProfile(true) + .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L) + .subAggregation(terms("tags").field(TAG_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD))) + .subAggregation(terms("strings").field(STRING_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD)) + .subAggregation(terms("tags").field(TAG_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD))))) + .get(); + assertSearchResponse(response); + Map profileResults = response.getProfileResults(); + assertThat(profileResults, notNullValue()); + assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries)); + for (ProfileShardResult profileShardResult : profileResults.values()) { + assertThat(profileShardResult, notNullValue()); + AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults(); + assertThat(aggProfileResults, notNullValue()); + List aggProfileResultsList = aggProfileResults.getProfileResults(); + assertThat(aggProfileResultsList, notNullValue()); + assertThat(aggProfileResultsList.size(), equalTo(1)); + ProfileResult histoAggResult = aggProfileResultsList.get(0); + assertThat(histoAggResult, notNullValue()); + assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName())); + assertThat(histoAggResult.getLuceneDescription(), equalTo("histo")); + assertThat(histoAggResult.getTime(), greaterThan(0L)); + Map histoBreakdown = histoAggResult.getTimeBreakdown(); + assertThat(histoBreakdown, notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(histoAggResult.getProfiledChildren().size(), equalTo(2)); + + ProfileResult tagsAggResult = histoAggResult.getProfiledChildren().get(0); + assertThat(tagsAggResult, notNullValue()); + assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName())); + assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags")); + assertThat(tagsAggResult.getTime(), greaterThan(0L)); + Map tagsBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(tagsBreakdown, notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(tagsAggResult.getProfiledChildren().size(), equalTo(2)); + + ProfileResult avgAggResult = tagsAggResult.getProfiledChildren().get(0); + assertThat(avgAggResult, notNullValue()); + assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName())); + assertThat(avgAggResult.getLuceneDescription(), equalTo("avg")); + assertThat(avgAggResult.getTime(), greaterThan(0L)); + Map avgBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(avgBreakdown, notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0)); + + ProfileResult maxAggResult = tagsAggResult.getProfiledChildren().get(1); + assertThat(maxAggResult, notNullValue()); + assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName())); + assertThat(maxAggResult.getLuceneDescription(), equalTo("max")); + assertThat(maxAggResult.getTime(), greaterThan(0L)); + Map maxBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(maxBreakdown, notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0)); + + ProfileResult stringsAggResult = histoAggResult.getProfiledChildren().get(1); + assertThat(stringsAggResult, notNullValue()); + assertThat(stringsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName())); + assertThat(stringsAggResult.getLuceneDescription(), equalTo("strings")); + assertThat(stringsAggResult.getTime(), greaterThan(0L)); + Map stringsBreakdown = stringsAggResult.getTimeBreakdown(); + assertThat(stringsBreakdown, notNullValue()); + assertThat(stringsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(stringsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(stringsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(stringsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(stringsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(stringsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(stringsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(stringsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(stringsAggResult.getProfiledChildren().size(), equalTo(3)); + + avgAggResult = stringsAggResult.getProfiledChildren().get(0); + assertThat(avgAggResult, notNullValue()); + assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName())); + assertThat(avgAggResult.getLuceneDescription(), equalTo("avg")); + assertThat(avgAggResult.getTime(), greaterThan(0L)); + avgBreakdown = stringsAggResult.getTimeBreakdown(); + assertThat(avgBreakdown, notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0)); + + maxAggResult = stringsAggResult.getProfiledChildren().get(1); + assertThat(maxAggResult, notNullValue()); + assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName())); + assertThat(maxAggResult.getLuceneDescription(), equalTo("max")); + assertThat(maxAggResult.getTime(), greaterThan(0L)); + maxBreakdown = stringsAggResult.getTimeBreakdown(); + assertThat(maxBreakdown, notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0)); + + tagsAggResult = stringsAggResult.getProfiledChildren().get(2); + assertThat(tagsAggResult, notNullValue()); + assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName())); + assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags")); + assertThat(tagsAggResult.getTime(), greaterThan(0L)); + tagsBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(tagsBreakdown, notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(tagsAggResult.getProfiledChildren().size(), equalTo(2)); + + avgAggResult = tagsAggResult.getProfiledChildren().get(0); + assertThat(avgAggResult, notNullValue()); + assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName())); + assertThat(avgAggResult.getLuceneDescription(), equalTo("avg")); + assertThat(avgAggResult.getTime(), greaterThan(0L)); + avgBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(avgBreakdown, notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0)); + + maxAggResult = tagsAggResult.getProfiledChildren().get(1); + assertThat(maxAggResult, notNullValue()); + assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName())); + assertThat(maxAggResult.getLuceneDescription(), equalTo("max")); + assertThat(maxAggResult.getTime(), greaterThan(0L)); + maxBreakdown = tagsAggResult.getTimeBreakdown(); + assertThat(maxBreakdown, notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L)); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue()); + assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L)); + assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0)); + } + } + + public void testNoProfile() { + SearchResponse response = client().prepareSearch("idx").setProfile(false) + .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L) + .subAggregation(terms("tags").field(TAG_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD))) + .subAggregation(terms("strings").field(STRING_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD)) + .subAggregation(terms("tags").field(TAG_FIELD) + .subAggregation(avg("avg").field(NUMBER_FIELD)) + .subAggregation(max("max").field(NUMBER_FIELD))))) + .get(); + assertSearchResponse(response); + Map profileResults = response.getProfileResults(); + assertThat(profileResults, notNullValue()); + assertThat(profileResults.size(), equalTo(0)); + } +} diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index 371aaadd3a7..b6935f021d4 100644 --- a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -85,8 +85,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shard : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shard.getValue()) { + for (Map.Entry shard : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shard.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -163,8 +163,9 @@ public class QueryProfilerIT extends ESIntegTestCase { nearlyEqual(vanillaMaxScore, profileMaxScore, 0.001)); } - assertThat("Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits [" - + vanillaResponse.getHits().totalHits() + "]", + assertThat( + "Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits [" + + vanillaResponse.getHits().totalHits() + "]", vanillaResponse.getHits().getTotalHits(), equalTo(profileResponse.getHits().getTotalHits())); SearchHit[] vanillaHits = vanillaResponse.getHits().getHits(); @@ -203,12 +204,12 @@ public class QueryProfilerIT extends ESIntegTestCase { .setSearchType(SearchType.QUERY_THEN_FETCH) .execute().actionGet(); - Map> p = resp.getProfileResults(); + Map p = resp.getProfileResults(); assertNotNull(p); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertEquals(result.getQueryName(), "TermQuery"); assertEquals(result.getLuceneDescription(), "field1:one"); @@ -250,12 +251,12 @@ public class QueryProfilerIT extends ESIntegTestCase { .setSearchType(SearchType.QUERY_THEN_FETCH) .execute().actionGet(); - Map> p = resp.getProfileResults(); + Map p = resp.getProfileResults(); assertNotNull(p); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertEquals(result.getQueryName(), "BooleanQuery"); assertEquals(result.getLuceneDescription(), "+field1:one +field1:two"); @@ -322,8 +323,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -374,8 +375,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -421,8 +422,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -468,8 +469,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -514,8 +515,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); @@ -569,8 +570,8 @@ public class QueryProfilerIT extends ESIntegTestCase { assertNotNull("Profile response element should not be null", resp.getProfileResults()); assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0)); - for (Map.Entry> shardResult : resp.getProfileResults().entrySet()) { - for (ProfileShardResult searchProfiles : shardResult.getValue()) { + for (Map.Entry shardResult : resp.getProfileResults().entrySet()) { + for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) { for (ProfileResult result : searchProfiles.getQueryResults()) { assertNotNull(result.getQueryName()); assertNotNull(result.getLuceneDescription()); diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java similarity index 96% rename from core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java rename to core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java index 7488dbceee7..ffad39bc3f2 100644 --- a/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java +++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java @@ -51,7 +51,7 @@ import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class ProfileTests extends ESTestCase { +public class QueryProfilerTests extends ESTestCase { static Directory dir; static IndexReader reader; @@ -90,7 +90,7 @@ public class ProfileTests extends ESTestCase { searcher.setProfiler(profiler); Query query = new TermQuery(new Term("foo", "bar")); searcher.search(query, 1); - List results = profiler.getQueryTree(); + List results = profiler.getTree(); assertEquals(1, results.size()); Map breakdown = results.get(0).getTimeBreakdown(); assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L)); @@ -109,7 +109,7 @@ public class ProfileTests extends ESTestCase { searcher.setProfiler(profiler); Query query = new TermQuery(new Term("foo", "bar")); searcher.search(query, 1, Sort.INDEXORDER); // scores are not needed - List results = profiler.getQueryTree(); + List results = profiler.getTree(); assertEquals(1, results.size()); Map breakdown = results.get(0).getTimeBreakdown(); assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L)); @@ -128,7 +128,7 @@ public class ProfileTests extends ESTestCase { searcher.setProfiler(profiler); Query query = new TermQuery(new Term("foo", "bar")); searcher.count(query); // will use index stats - List results = profiler.getQueryTree(); + List results = profiler.getTree(); assertEquals(0, results.size()); long rewriteTime = profiler.getRewriteTime(); @@ -144,7 +144,7 @@ public class ProfileTests extends ESTestCase { searcher.setProfiler(profiler); Query query = new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random()); searcher.count(query); - List results = profiler.getQueryTree(); + List results = profiler.getTree(); assertEquals(1, results.size()); Map breakdown = results.get(0).getTimeBreakdown(); assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L)); diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 9751df26329..e94fe21a170 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -443,6 +443,6 @@ public class SimpleSearchIT extends ESIntegTestCase { assertThat(e.toString(), containsString("Rescore window [" + windowSize + "] is too large. It must " + "be less than [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY))); assertThat(e.toString(), containsString( - "This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting.")); + "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting.")); } } diff --git a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java index ad93d14f21f..4a27dcb5964 100644 --- a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java +++ b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java @@ -35,7 +35,6 @@ import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; import java.util.List; import java.util.ArrayList; -import java.util.Set; import java.util.HashSet; import java.util.concurrent.ExecutionException; @@ -43,7 +42,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.startsWith; public class SearchSliceIT extends ESIntegTestCase { @@ -71,7 +69,8 @@ public class SearchSliceIT extends ESIntegTestCase { .endObject().string(); int numberOfShards = randomIntBetween(1, 7); assertAcked(client().admin().indices().prepareCreate("test") - .setSettings("number_of_shards", numberOfShards) + .setSettings("number_of_shards", numberOfShards, + "index.max_slices_per_scroll", 10000) .addMapping("type", mapping)); ensureGreen(); diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java new file mode 100644 index 00000000000..d35d6ce82c2 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tasks; + +import org.elasticsearch.client.Requests; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +/** + * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. + */ +public class PersistedTaskInfoTests extends ESTestCase { + public void testBinaryRoundTrip() throws IOException { + NamedWriteableRegistry registry = new NamedWriteableRegistry(); + registry.register(Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new); + PersistedTaskInfo result = randomTaskResult(); + PersistedTaskInfo read; + try (BytesStreamOutput out = new BytesStreamOutput()) { + result.writeTo(out); + try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry)) { + read = new PersistedTaskInfo(in); + } + } catch (IOException e) { + throw new IOException("Error processing [" + result + "]", e); + } + assertEquals(result, read); + } + + public void testXContentRoundTrip() throws IOException { + /* + * Note that this round trip isn't 100% perfect - status will always be read as RawTaskStatus. Since this test uses RawTaskStatus + * as the status we randomly generate then we can assert the round trip with .equals. + */ + PersistedTaskInfo result = randomTaskResult(); + PersistedTaskInfo read; + try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + try (XContentBuilder shuffled = shuffleXContent(builder); + XContentParser parser = XContentHelper.createParser(shuffled.bytes())) { + read = PersistedTaskInfo.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT); + } + } catch (IOException e) { + throw new IOException("Error processing [" + result + "]", e); + } + assertEquals(result, read); + } + + private static PersistedTaskInfo randomTaskResult() throws IOException { + switch (between(0, 2)) { + case 0: + return new PersistedTaskInfo(randomTaskInfo()); + case 1: + return new PersistedTaskInfo(randomTaskInfo(), new RuntimeException("error")); + case 2: + return new PersistedTaskInfo(randomTaskInfo(), randomTaskActionResult()); + default: + throw new UnsupportedOperationException("Unsupported random TaskResult constructor"); + } + } + + private static TaskInfo randomTaskInfo() throws IOException { + TaskId taskId = randomTaskId(); + String type = randomAsciiOfLength(5); + String action = randomAsciiOfLength(5); + Task.Status status = randomBoolean() ? randomRawTaskStatus() : null; + String description = randomBoolean() ? randomAsciiOfLength(5) : null; + long startTime = randomLong(); + long runningTimeNanos = randomLong(); + boolean cancellable = randomBoolean(); + TaskId parentTaskId = randomBoolean() ? TaskId.EMPTY_TASK_ID : randomTaskId(); + return new TaskInfo(taskId, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId); + } + + private static TaskId randomTaskId() { + return new TaskId(randomAsciiOfLength(5), randomLong()); + } + + private static RawTaskStatus randomRawTaskStatus() throws IOException { + try (XContentBuilder builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent())) { + builder.startObject(); + int fields = between(0, 10); + for (int f = 0; f < fields; f++) { + builder.field(randomAsciiOfLength(5), randomAsciiOfLength(5)); + } + builder.endObject(); + return new RawTaskStatus(builder.bytes()); + } + } + + private static ToXContent randomTaskActionResult() { + Map result = new TreeMap<>(); + int fields = between(0, 10); + for (int f = 0; f < fields; f++) { + result.put(randomAsciiOfLength(5), randomAsciiOfLength(5)); + } + return new ToXContent() { + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Results in Elasticsearch never output a leading startObject. There isn't really a good reason, they just don't. + for (Map.Entry entry : result.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + return builder; + } + }; + } +} diff --git a/core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip b/core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip new file mode 100644 index 00000000000..9590f8dbd66 Binary files /dev/null and b/core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip differ diff --git a/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1 deleted file mode 100644 index b581809a004..00000000000 --- a/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -53953c1a9b097f83209c84a422cf8f9d271f47c1 \ No newline at end of file diff --git a/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..bb4cc98e068 --- /dev/null +++ b/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +bf73c03e6b83f8e696133f40b9b1fc3381750149 \ No newline at end of file diff --git a/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1 deleted file mode 100644 index 5433f09c993..00000000000 --- a/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3647088603be84b8f4916ef86954e3336b98d254 \ No newline at end of file diff --git a/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..a8eb4883e15 --- /dev/null +++ b/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +8bc384f55faf99b6d6cee6f34df4fbd3145afb4d \ No newline at end of file diff --git a/distribution/licenses/lucene-core-6.0.1.jar.sha1 b/distribution/licenses/lucene-core-6.0.1.jar.sha1 deleted file mode 100644 index 6bcd7fc87f4..00000000000 --- a/distribution/licenses/lucene-core-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -40ccd40bec54266a10aa1f81c565914ede8c0ca0 \ No newline at end of file diff --git a/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..012fea19fdc --- /dev/null +++ b/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +fe19e7558440e10db4bd7150931dff6a7cf73243 \ No newline at end of file diff --git a/distribution/licenses/lucene-grouping-6.0.1.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.1.jar.sha1 deleted file mode 100644 index b132acc9112..00000000000 --- a/distribution/licenses/lucene-grouping-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -010daaae60227fbe719ca95e9b6fcdb5c38d4eba \ No newline at end of file diff --git a/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..3b3338ffd2f --- /dev/null +++ b/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +c3f0de4cdd185d23bce66c580d9c12adb98182a5 \ No newline at end of file diff --git a/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1 deleted file mode 100644 index 95d4b3edab9..00000000000 --- a/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -65d74c3642e6a86ba905045473b17cc84826527e \ No newline at end of file diff --git a/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..255812ed5fb --- /dev/null +++ b/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +ffb7087267bb6076b00c90f97ee36ebe23ea0662 \ No newline at end of file diff --git a/distribution/licenses/lucene-join-6.0.1.jar.sha1 b/distribution/licenses/lucene-join-6.0.1.jar.sha1 deleted file mode 100644 index 07392cf260f..00000000000 --- a/distribution/licenses/lucene-join-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2105e2826ce93d1f764e5a0a3afa9ee461d556c1 \ No newline at end of file diff --git a/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..4231857b4e7 --- /dev/null +++ b/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +d071ad17bed58b3267f6fa0b2a8211f8fe18c912 \ No newline at end of file diff --git a/distribution/licenses/lucene-memory-6.0.1.jar.sha1 b/distribution/licenses/lucene-memory-6.0.1.jar.sha1 deleted file mode 100644 index b9820103d3f..00000000000 --- a/distribution/licenses/lucene-memory-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e2cde0688e487a27d08df0c2d81d492b1f4cdc2a \ No newline at end of file diff --git a/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..19aa64ccd80 --- /dev/null +++ b/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +f5e9b6eefe580a7f65276aca3192ca5796332509 \ No newline at end of file diff --git a/distribution/licenses/lucene-misc-6.0.1.jar.sha1 b/distribution/licenses/lucene-misc-6.0.1.jar.sha1 deleted file mode 100644 index 2670ab628df..00000000000 --- a/distribution/licenses/lucene-misc-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e6e59996fc324319d695e41cf25e30e5f1e4c182 \ No newline at end of file diff --git a/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..8480fcc3a49 --- /dev/null +++ b/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +6b84a79c37b01197130cceb65e5573794f073df1 \ No newline at end of file diff --git a/distribution/licenses/lucene-queries-6.0.1.jar.sha1 b/distribution/licenses/lucene-queries-6.0.1.jar.sha1 deleted file mode 100644 index acaa53f1f8e..00000000000 --- a/distribution/licenses/lucene-queries-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -09b0e5862a676ff9e55a1bc6ca37ad578a25cb38 \ No newline at end of file diff --git a/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..a3ed70c12af --- /dev/null +++ b/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +a9d51b77395dfdd7e6c4cf8c8506ebca5e1bb374 \ No newline at end of file diff --git a/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1 deleted file mode 100644 index 48c91d68f44..00000000000 --- a/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -24f7ba0707aa01be2dd7749adff1659262be8f33 \ No newline at end of file diff --git a/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..6bf59bc00e7 --- /dev/null +++ b/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +e322f004e574df119ba08dd8751a743422a46724 \ No newline at end of file diff --git a/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1 deleted file mode 100644 index ef843328aa0..00000000000 --- a/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0faf4c0d7e0adb6fccd830a2d5797d4176b579fe \ No newline at end of file diff --git a/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..79567427105 --- /dev/null +++ b/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +c7cb119652c906adcdf7fe64445c76d057329d63 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.1.jar.sha1 deleted file mode 100644 index 25e7232ac14..00000000000 --- a/distribution/licenses/lucene-spatial-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4d94d006251c904de3f1503c64746400877d6fa3 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..2b06178aaad --- /dev/null +++ b/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +ca6c17fe31884e968ae63fd475ce6532b767c7fa \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1 deleted file mode 100644 index d421e1b053c..00000000000 --- a/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3de19dbdb889fe87791dae291ac3b340586854c4 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..9c487b7746f --- /dev/null +++ b/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +49235405e40757474aaa9e8e54946b67fe2a01d9 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1 deleted file mode 100644 index 348f501bb52..00000000000 --- a/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5b1b7a754e83e2d58a819afa279b20b08b48c9c1 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..1eaab9f9955 --- /dev/null +++ b/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +39f6b29c428327860c1a342bd57800e79ad92ef5 \ No newline at end of file diff --git a/distribution/licenses/lucene-suggest-6.0.1.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.1.jar.sha1 deleted file mode 100644 index 2cb6272d826..00000000000 --- a/distribution/licenses/lucene-suggest-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55886bdaf16ecc6948e94b527837eaa1f16fe988 \ No newline at end of file diff --git a/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..948dfc4c2b9 --- /dev/null +++ b/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +4e9f713d34fd4208bf308ac59132216f96521f13 \ No newline at end of file diff --git a/docs/build.gradle b/docs/build.gradle index 239eb81e820..660755a1c65 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -22,6 +22,29 @@ apply plugin: 'elasticsearch.docs-test' integTest { cluster { setting 'script.inline', 'true' + setting 'script.stored', 'true' + Closure configFile = { + extraConfigFile it, "src/test/cluster/config/$it" + } + configFile 'scripts/my_script.js' + configFile 'scripts/my_script.py' + configFile 'userdict_ja.txt' + configFile 'KeywordTokenizer.rbbi' + } +} + +// Build the cluser with all plugins +project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj -> + /* Skip repositories. We just aren't going to be able to test them so it + * doesn't make sense to waste time installing them. */ + if (subproj.path.startsWith(':plugins:repository-')) { + return + } + integTest { + cluster { + // We need a non-decorated project object, so we lookup the project by path + plugin subproj.name, project(subproj.path) + } } } @@ -30,8 +53,6 @@ buildRestTests.docs = fileTree(projectDir) { exclude 'build.gradle' // That is where the snippets go, not where they come from! exclude 'build' - // Remove plugins because they aren't installed during this test. Yet? - exclude 'plugins' // This file simply doesn't pass yet. We should figure out how to fix it. exclude 'reference/modules/snapshots.asciidoc' } diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index 815e6285235..3dbe7b2e354 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -161,6 +161,8 @@ PUT icu_sample } } +GET _cluster/health?wait_for_status=yellow + POST icu_sample/_analyze?analyzer=my_analyzer&text=Elasticsearch. Wow! -------------------------------------------------- // CONSOLE @@ -169,7 +171,6 @@ The above `analyze` request returns the following: [source,js] -------------------------------------------------- -# Result { "tokens": [ { @@ -182,7 +183,7 @@ The above `analyze` request returns the following: ] } -------------------------------------------------- - +// TESTRESPONSE [[analysis-icu-normalization]] ==== ICU Normalization Token Filter @@ -253,7 +254,7 @@ PUT icu_sample "analysis": { "analyzer": { "folded": { - "tokenizer": "icu", + "tokenizer": "icu_tokenizer", "filter": [ "icu_folding" ] @@ -359,6 +360,8 @@ PUT /my_index } } +GET _cluster/health?wait_for_status=yellow + GET _search <3> { "query": { @@ -478,6 +481,8 @@ PUT icu_sample } } +GET _cluster/health?wait_for_status=yellow + GET icu_sample/_analyze?analyzer=latin { "text": "你好" <2> diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index f29d13c2867..b0d6c1eb962 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -172,6 +172,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=東京スカイツリー -------------------------------------------------- // CONSOLE @@ -224,6 +226,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=飲み -------------------------------------------------- // CONSOLE @@ -282,6 +286,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=寿司がおいしいね -------------------------------------------------- @@ -354,6 +360,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=katakana_analyzer&text=寿司 <1> POST kuromoji_sample/_analyze?analyzer=romaji_analyzer&text=寿司 <2> @@ -405,6 +413,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=コピー <1> POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=サーバー <2> @@ -454,7 +464,9 @@ PUT kuromoji_sample } } -POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=ストップは消える +GET _cluster/health?wait_for_status=yellow + +POST kuromoji_sample/_analyze?analyzer=analyzer_with_ja_stop&text=ストップは消える -------------------------------------------------- // CONSOLE @@ -500,6 +512,8 @@ PUT kuromoji_sample } } +GET _cluster/health?wait_for_status=yellow + POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇 -------------------------------------------------- @@ -518,4 +532,3 @@ POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇 } ] } -------------------------------------------------- - diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc index f37547d9f75..b44218b7521 100644 --- a/docs/plugins/analysis-phonetic.asciidoc +++ b/docs/plugins/analysis-phonetic.asciidoc @@ -79,6 +79,8 @@ PUT phonetic_sample } } +GET _cluster/health?wait_for_status=yellow + POST phonetic_sample/_analyze?analyzer=my_analyzer&text=Joe Bloggs <1> -------------------------------------------------- // CONSOLE @@ -116,5 +118,3 @@ supported: be guessed. Accepts: `any`, `comomon`, `cyrillic`, `english`, `french`, `german`, `hebrew`, `hungarian`, `polish`, `romanian`, `russian`, `spanish`. - - diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc index bfee18a0a89..b7f858d6b4e 100644 --- a/docs/plugins/lang-javascript.asciidoc +++ b/docs/plugins/lang-javascript.asciidoc @@ -53,8 +53,6 @@ you can use JavaScript as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -96,8 +94,6 @@ you can use JavaScript as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -129,7 +125,6 @@ GET test/_search } } } - ---- // CONSOLE @@ -145,7 +140,7 @@ You can save your scripts to a file in the `config/scripts/` directory on every node. The `.javascript` file suffix identifies the script as containing JavaScript: -First, save this file as `config/scripts/my_script.javascript` on every node +First, save this file as `config/scripts/my_script.js` on every node in the cluster: [source,js] @@ -157,8 +152,6 @@ then use the script as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -185,9 +178,7 @@ GET test/_search } } } - ---- // CONSOLE <1> The function score query retrieves the script with filename `my_script.javascript`. - diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc index af2fc8a3a39..3920ff0ab0a 100644 --- a/docs/plugins/lang-python.asciidoc +++ b/docs/plugins/lang-python.asciidoc @@ -52,8 +52,6 @@ you can use Python as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -95,8 +93,6 @@ you can use Python as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -156,8 +152,6 @@ then use the script as follows: [source,js] ---- -DELETE test - PUT test/doc/1 { "num": 1.0 @@ -184,9 +178,7 @@ GET test/_search } } } - ---- // CONSOLE <1> The function score query retrieves the script with filename `my_script.py`. - diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc index 570d05a7513..86e80cb1a26 100644 --- a/docs/plugins/mapper-attachments.asciidoc +++ b/docs/plugins/mapper-attachments.asciidoc @@ -57,12 +57,13 @@ Index a new document populated with a `base64`-encoded attachment: [source,js] -------------------------- -POST /trying-out-mapper-attachments/person/1 +POST /trying-out-mapper-attachments/person/1?refresh { "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" } -------------------------- // CONSOLE +// TEST[continued] Search for the document using words in the attachment: @@ -76,8 +77,35 @@ POST /trying-out-mapper-attachments/person/_search }}} -------------------------- // CONSOLE +// TEST[continued] -If you get a hit for your indexed document, the plugin should be installed and working. +If you get a hit for your indexed document, the plugin should be installed and working. It'll look like: + +[source,js] +-------------------------- +{ + "timed_out": false, + "took": 53, + "hits": { + "total": 1, + "max_score": 0.3125, + "hits": [ + { + "_score": 0.3125, + "_index": "trying-out-mapper-attachments", + "_type": "person", + "_id": "1", + "_source": { + "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" + } + } + ] + }, + "_shards": ... +} +-------------------------- +// TESTRESPONSE[s/"took": 53/"took": "$body.took"/] +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards"/] [[mapper-attachments-usage]] ==== Usage @@ -87,13 +115,14 @@ Using the attachment type is simple, in your mapping JSON, simply set a certain [source,js] -------------------------- PUT /test -PUT /test/person/_mapping { + "mappings": { "person" : { - "properties" : { - "my_attachment" : { "type" : "attachment" } - } + "properties" : { + "my_attachment" : { "type" : "attachment" } + } } + } } -------------------------- // CONSOLE @@ -146,25 +175,40 @@ in the mappings. For example: [source,js] -------------------------- -PUT /test/person/_mapping +PUT /test { - "person" : { - "properties" : { - "file" : { - "type" : "attachment", - "fields" : { - "content" : {"index" : "no"}, - "title" : {"store" : "yes"}, - "date" : {"store" : "yes"}, - "author" : {"analyzer" : "myAnalyzer"}, - "keywords" : {"store" : "yes"}, - "content_type" : {"store" : "yes"}, - "content_length" : {"store" : "yes"}, - "language" : {"store" : "yes"} - } - } + "settings": { + "index": { + "analysis": { + "analyzer": { + "my_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["standard"] + } } + } } + }, + "mappings": { + "person" : { + "properties" : { + "file" : { + "type" : "attachment", + "fields" : { + "content" : {"index" : "no"}, + "title" : {"store" : "yes"}, + "date" : {"store" : "yes"}, + "author" : {"analyzer" : "my_analyzer"}, + "keywords" : {"store" : "yes"}, + "content_type" : {"store" : "yes"}, + "content_length" : {"store" : "yes"}, + "language" : {"store" : "yes"} + } + } + } + } + } } -------------------------- // CONSOLE @@ -179,7 +223,6 @@ If you need to query on metadata fields, use the attachment field name dot the m [source,js] -------------------------- -DELETE /test PUT /test PUT /test/person/_mapping { @@ -300,7 +343,6 @@ If you want to highlight your attachment content, you will need to set `"store": [source,js] -------------------------- -DELETE /test PUT /test PUT /test/person/_mapping { diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc index 4111a15ef6c..800a640890a 100644 --- a/docs/plugins/mapper-size.asciidoc +++ b/docs/plugins/mapper-size.asciidoc @@ -52,8 +52,7 @@ PUT my_index -------------------------- // CONSOLE -The value of the `_size` field is accessible in queries, aggregations, scripts, -and when sorting: +The value of the `_size` field is accessible in queries: [source,js] -------------------------- @@ -76,33 +75,10 @@ GET my_index/_search "gt": 10 } } - }, - "aggs": { - "Sizes": { - "terms": { - "field": "_size", <2> - "size": 10 - } - } - }, - "sort": [ - { - "_size": { <3> - "order": "desc" - } - } - ], - "script_fields": { - "Size": { - "script": "doc['_size']" <4> - } } } -------------------------- // CONSOLE +// TEST[continued] <1> Querying on the `_size` field -<2> Aggregating on the `_size` field -<3> Sorting on the `_size` field -<4> Accessing the `_size` field in scripts (inline scripts must be modules-security-scripting.html#enable-dynamic-scripting[enabled] for this example to work) - diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index f338728a35d..03466f0c643 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -168,6 +168,7 @@ PUT _snapshot/my_backup4 } ---- // CONSOLE +// TEST[skip:we don't have azure setup while testing this] Example using Java: @@ -207,4 +208,3 @@ be a valid DNS name, conforming to the following naming rules: permitted in container names. * All letters in a container name must be lowercase. * Container names must be from 3 through 63 characters long. - diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc index ea2cb99ee9b..5b2c633f23e 100644 --- a/docs/plugins/repository-gcs.asciidoc +++ b/docs/plugins/repository-gcs.asciidoc @@ -82,7 +82,7 @@ https://console.cloud.google.com/compute/[Compute Engine console]. To indicate that a repository should use the built-in authentication, the repository `service_account` setting must be set to `_default_`: -[source,json] +[source,js] ---- PUT _snapshot/my_gcs_repository_on_compute_engine { @@ -94,6 +94,7 @@ PUT _snapshot/my_gcs_repository_on_compute_engine } ---- // CONSOLE +// TEST[skip:we don't have gcs setup while testing this] NOTE: The Compute Engine VM must be allowed to use the Storage service. This can be done only at VM creation time, when "Storage" access can be configured to "Read/Write" permission. Check your @@ -115,7 +116,7 @@ To create a service account file: A service account file looks like this: -[source,json] +[source,js] ---- { "type": "service_account", @@ -136,7 +137,7 @@ every node of the cluster. To indicate that a repository should use a service account file: -[source,json] +[source,js] ---- PUT _snapshot/my_gcs_repository { @@ -148,7 +149,7 @@ PUT _snapshot/my_gcs_repository } ---- // CONSOLE - +// TEST[skip:we don't have gcs setup while testing this] [[repository-gcs-bucket-permission]] ===== Set Bucket Permission @@ -168,7 +169,7 @@ The service account used to access the bucket must have the "Writer" access to t Once everything is installed and every node is started, you can create a new repository that uses Google Cloud Storage to store snapshots: -[source,json] +[source,js] ---- PUT _snapshot/my_gcs_repository { @@ -180,6 +181,7 @@ PUT _snapshot/my_gcs_repository } ---- // CONSOLE +// TEST[skip:we don't have gcs setup while testing this] The following settings are supported: diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 82485b6d9d0..f086158954c 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -149,6 +149,7 @@ PUT _snapshot/my_s3_repository } ---- // CONSOLE +// TEST[skip:we don't have s3 set up while testing this] The following settings are supported: diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc index b550b240b75..9963b48fb04 100644 --- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc @@ -117,15 +117,9 @@ precision:: Optional. The string length of the geohashes used to define size:: Optional. The maximum number of geohash buckets to return (defaults to 10,000). When results are trimmed, buckets are prioritised based on the volumes of documents they contain. - A value of `0` will return all buckets that - contain a hit, use with caution as this could use a lot of CPU - and network bandwidth if there are many buckets. shard_size:: Optional. To allow for more accurate counting of the top cells returned in the final result the aggregation defaults to returning `max(10,(size x number-of-shards))` buckets from each shard. If this heuristic is undesirable, the number considered from each shard can be over-ridden using this parameter. - A value of `0` makes the shard size unlimited. - - diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc index 80c747e61a5..61deffeccd2 100644 --- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc @@ -224,12 +224,12 @@ are presented unstemmed, highlighted, with the right case, in the right order an ==== Custom background sets Ordinarily, the foreground set of documents is "diffed" against a background set of all the documents in your index. -However, sometimes it may prove useful to use a narrower background set as the basis for comparisons. -For example, a query on documents relating to "Madrid" in an index with content from all over the world might reveal that "Spanish" -was a significant term. This may be true but if you want some more focused terms you could use a `background_filter` -on the term 'spain' to establish a narrower set of documents as context. With this as a background "Spanish" would now -be seen as commonplace and therefore not as significant as words like "capital" that relate more strongly with Madrid. -Note that using a background filter will slow things down - each term's background frequency must now be derived on-the-fly from filtering posting lists rather than reading the index's pre-computed count for a term. +However, sometimes it may prove useful to use a narrower background set as the basis for comparisons. +For example, a query on documents relating to "Madrid" in an index with content from all over the world might reveal that "Spanish" +was a significant term. This may be true but if you want some more focused terms you could use a `background_filter` +on the term 'spain' to establish a narrower set of documents as context. With this as a background "Spanish" would now +be seen as commonplace and therefore not as significant as words like "capital" that relate more strongly with Madrid. +Note that using a background filter will slow things down - each term's background frequency must now be derived on-the-fly from filtering posting lists rather than reading the index's pre-computed count for a term. ==== Limitations @@ -274,7 +274,7 @@ The scores are derived from the doc frequencies in _foreground_ and _background_ ===== mutual information Mutual information as described in "Information Retrieval", Manning et al., Chapter 13.5.1 can be used as significance score by adding the parameter - + [source,js] -------------------------------------------------- @@ -283,9 +283,9 @@ Mutual information as described in "Information Retrieval", Manning et al., Chap } -------------------------------------------------- -Mutual information does not differentiate between terms that are descriptive for the subset or for documents outside the subset. The significant terms therefore can contain terms that appear more or less frequent in the subset than outside the subset. To filter out the terms that appear less often in the subset than in documents outside the subset, `include_negatives` can be set to `false`. +Mutual information does not differentiate between terms that are descriptive for the subset or for documents outside the subset. The significant terms therefore can contain terms that appear more or less frequent in the subset than outside the subset. To filter out the terms that appear less often in the subset than in documents outside the subset, `include_negatives` can be set to `false`. -Per default, the assumption is that the documents in the bucket are also contained in the background. If instead you defined a custom background filter that represents a different set of documents that you want to compare to, set +Per default, the assumption is that the documents in the bucket are also contained in the background. If instead you defined a custom background filter that represents a different set of documents that you want to compare to, set [source,js] -------------------------------------------------- @@ -296,7 +296,7 @@ Per default, the assumption is that the documents in the bucket are also contain ===== Chi square Chi square as described in "Information Retrieval", Manning et al., Chapter 13.5.2 can be used as significance score by adding the parameter - + [source,js] -------------------------------------------------- @@ -309,7 +309,7 @@ Chi square behaves like mutual information and can be configured with the same p ===== google normalized distance Google normalized distance as described in "The Google Similarity Distance", Cilibrasi and Vitanyi, 2007 (http://arxiv.org/pdf/cs/0412098v3.pdf) can be used as significance score by adding the parameter - + [source,js] -------------------------------------------------- @@ -317,7 +317,7 @@ Google normalized distance as described in "The Google Similarity Distance", Ci } -------------------------------------------------- -`gnd` also accepts the `background_is_superset` parameter. +`gnd` also accepts the `background_is_superset` parameter. ===== Percentage @@ -328,7 +328,7 @@ The benefit of this heuristic is that the scoring logic is simple to explain to It would be hard for a seasoned boxer to win a championship if the prize was awarded purely on the basis of percentage of fights won - by these rules a newcomer with only one fight under his belt would be impossible to beat. Multiple observations are typically required to reinforce a view so it is recommended in these cases to set both `min_doc_count` and `shard_min_doc_count` to a higher value such as 10 in order to filter out the low-frequency terms that otherwise take precedence. - + [source,js] -------------------------------------------------- @@ -348,7 +348,7 @@ If none of the above measures suits your usecase than another option is to imple ===== scripted Customized scores can be implemented via a script: - + [source,js] -------------------------------------------------- @@ -357,7 +357,7 @@ Customized scores can be implemented via a script: } -------------------------------------------------- -Scripts can be inline (as in above example), indexed or stored on disk. For details on the options, see <>. +Scripts can be inline (as in above example), indexed or stored on disk. For details on the options, see <>. Available parameters in the script are @@ -374,9 +374,7 @@ default, the node coordinating the search process will request each shard to pro and once all shards respond, it will reduce the results to the final list that will then be returned to the client. If the number of unique terms is greater than `size`, the returned list can be slightly off and not accurate (it could be that the term counts are slightly off and it could even be that a term that should have been in the top -size buckets was not returned). - -If set to `0`, the `size` will be set to `Integer.MAX_VALUE`. +size buckets was not returned). To ensure better accuracy a multiple of the final `size` is used as the number of terms to request from each shard using a heuristic based on the number of shards. To take manual control of this setting the `shard_size` parameter @@ -386,12 +384,9 @@ Low-frequency terms can turn out to be the most interesting ones once all result significant_terms aggregation can produce higher-quality results when the `shard_size` parameter is set to values significantly higher than the `size` setting. This ensures that a bigger volume of promising candidate terms are given a consolidated review by the reducing node before the final selection. Obviously large candidate term lists -will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. +will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. -If set to `0`, the `shard_size` will be set to `Integer.MAX_VALUE`. - - NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will override it and reset it to be equal to `size`. @@ -439,7 +434,7 @@ WARNING: Setting `min_doc_count` to `1` is generally not advised as it tends to The default source of statistical information for background term frequencies is the entire index and this scope can be narrowed through the use of a `background_filter` to focus in on significant terms within a narrower -context: +context: [source,js] -------------------------------------------------- @@ -449,7 +444,7 @@ context: }, "aggs" : { "tags" : { - "significant_terms" : { + "significant_terms" : { "field" : "tag", "background_filter": { "term" : { "text" : "spain"} @@ -460,9 +455,9 @@ context: } -------------------------------------------------- -The above filter would help focus in on terms that were peculiar to the city of Madrid rather than revealing -terms like "Spanish" that are unusual in the full index's worldwide context but commonplace in the subset of documents containing the -word "Spain". +The above filter would help focus in on terms that were peculiar to the city of Madrid rather than revealing +terms like "Spanish" that are unusual in the full index's worldwide context but commonplace in the subset of documents containing the +word "Spain". WARNING: Use of background filters will slow the query as each term's postings must be filtered to determine a frequency @@ -482,7 +477,7 @@ There are different mechanisms by which terms aggregations can be executed: - by using field values directly in order to aggregate data per-bucket (`map`) - by using ordinals of the field and preemptively allocating one bucket per ordinal value (`global_ordinals`) - by using ordinals of the field and dynamically allocating one bucket per ordinal value (`global_ordinals_hash`) - + Elasticsearch tries to have sensible defaults so this is something that generally doesn't need to be configured. `map` should only be considered when very few documents match a query. Otherwise the ordinals-based execution modes @@ -514,4 +509,3 @@ in inner aggregations. <1> the possible values are `map`, `global_ordinals` and `global_ordinals_hash` Please note that Elasticsearch will ignore this execution hint if it is not applicable. - diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 1d9a65a7428..1c88f0a06d8 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -56,7 +56,7 @@ default, the node coordinating the search process will request each shard to pro and once all shards respond, it will reduce the results to the final list that will then be returned to the client. This means that if the number of unique terms is greater than `size`, the returned list is slightly off and not accurate (it could be that the term counts are slightly off and it could even be that a term that should have been in the top -size buckets was not returned). If set to `0`, the `size` will be set to `Integer.MAX_VALUE`. +size buckets was not returned). [[search-aggregations-bucket-terms-aggregation-approximate-counts]] ==== Document counts are approximate @@ -149,14 +149,12 @@ The `shard_size` parameter can be used to minimize the extra work that comes wi it will determine how many terms the coordinating node will request from each shard. Once all the shards responded, the coordinating node will then reduce them to a final result which will be based on the `size` parameter - this way, one can increase the accuracy of the returned terms and avoid the overhead of streaming a big list of buckets back to -the client. If set to `0`, the `shard_size` will be set to `Integer.MAX_VALUE`. +the client. NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will override it and reset it to be equal to `size`. -It is possible to not limit the number of terms that are returned by setting `size` to `0`. Don't use this -on high-cardinality fields as this will kill both your CPU since terms need to be return sorted, and your network. The default `shard_size` is a multiple of the `size` parameter which is dependant on the number of shards. @@ -443,7 +441,7 @@ Generating the terms using a script: "aggs" : { "genders" : { "terms" : { - "script" : { + "script" : { "inline": "doc['gender'].value" "lang": "painless" } @@ -485,9 +483,9 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter. "genders" : { "terms" : { "field" : "gender", - "script" : { + "script" : { "inline" : "'Gender: ' +_value" - "lang" : "painless" + "lang" : "painless" } } } @@ -710,4 +708,4 @@ had a value. } -------------------------------------------------- -<1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`. +<1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`. diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc index 2d5741c2b9e..bd6000c3de7 100644 --- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc @@ -148,7 +148,7 @@ The `pattern` analyzer accepts the following parameters: `flags`:: Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags]. - lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`. + Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`. `lowercase`:: diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc index 72adefa5aec..24cd203ae26 100644 --- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc +++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc @@ -21,6 +21,11 @@ The `pattern_replace` character filter accepts the following parameters: `$1`..`$9` syntax, as explained http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#appendReplacement-java.lang.StringBuffer-java.lang.String-[here]. +`flags`:: + + Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags]. + Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`. + [float] === Example configuration diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc index b53d8de93a1..5720227f9f0 100644 --- a/docs/reference/cluster/tasks.asciidoc +++ b/docs/reference/cluster/tasks.asciidoc @@ -58,15 +58,26 @@ The result will look similar to the following: } -------------------------------------------------- -It is also possible to retrieve information for a particular task, or all children of a particular -tasks using the following two commands: +It is also possible to retrieve information for a particular task: [source,js] -------------------------------------------------- -GET _tasks/taskId:1 -GET _tasks?parent_task_id=parentTaskId:1 +GET _tasks/taskId:1 <1> -------------------------------------------------- // CONSOLE +// TEST[catch:missing] + +<1> This will return a 404 if the task isn't found. + +Or to retrieve all children of a particular task: + +[source,js] +-------------------------------------------------- +GET _tasks?parent_task_id=parentTaskId:1 <1> +-------------------------------------------------- +// CONSOLE + +<1> This won't return a 404 if the parent isn't found. The task API can be also used to wait for completion of a particular task. The following call will block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is completed. @@ -76,6 +87,16 @@ block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is GET _tasks/oTUltX4IQMOUUVeiohTt8A:12345?wait_for_completion=true&timeout=10s -------------------------------------------------- // CONSOLE +// TEST[catch:missing] + +You can also wait for all tasks for certain action types to finish. This +command will wait for all `reindex` tasks to finish: + +[source,js] +-------------------------------------------------- +GET _tasks?actions=*reindex&wait_for_completion=true&timeout=10s +-------------------------------------------------- +// CONSOLE Tasks can be also listed using _cat version of the list tasks command, which accepts the same arguments as the standard list tasks command. diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index 1562d8c515d..8b37e0a1220 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -149,10 +149,11 @@ to be refreshed. If the request contains `wait_for_completion=false` then Elasticsearch will perform some preflight checks, launch the request, and then return a `task` -which can be used with <> to cancel -or get the status of the task. For now, once the request is finished the task -is gone and the only place to look for the ultimate result of the task is in -the Elasticsearch log file. This will be fixed soon. +which can be used with <> +to cancel or get the status of the task. Elasticsearch will also create a +record of this task as a document at `.tasks/task/${taskId}`. This is yours +to keep or remove as you see fit. When you are done with it, delete it so +Elasticsearch can reclaim the space it uses. `consistency` controls how many copies of a shard must respond to each write request. `timeout` controls how long each write request waits for unavailable @@ -222,7 +223,7 @@ from aborting the operation. [[docs-delete-by-query-task-api]] === Works with the Task API -While Delete By Query is running you can fetch their status using the +You can fetch the status of any running delete-by-query requests with the <>: [source,js] @@ -277,6 +278,22 @@ of operations that the reindex expects to perform. You can estimate the progress by adding the `updated`, `created`, and `deleted` fields. The request will finish when their sum is equal to the `total` field. +With the task id you can look up the task directly: + +[source,js] +-------------------------------------------------- +GET /_tasks/taskId:1 +-------------------------------------------------- +// CONSOLE +// TEST[catch:missing] + +The advantage of this API is that it integrates with `wait_for_completion=false` +to transparently return the status of completed tasks. If the task is completed +and `wait_for_completion=false` was set on it them it'll come back with a +`results` or an `error` field. The cost of this feature is the document that +`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to +you to delete that document. + [float] [[docs-delete-by-query-cancel-task-api]] diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc index 3e5153341c8..dd829e19bc3 100644 --- a/docs/reference/docs/refresh.asciidoc +++ b/docs/reference/docs/refresh.asciidoc @@ -28,6 +28,7 @@ APIs that support it. Take no refresh related actions. The changes made by this request will be made visible at some point after the request returns. +[float] === Choosing which setting to use Unless you have a good reason to wait for the change to become visible always @@ -60,6 +61,7 @@ refresh immediately, `refresh=true` will affect other ongoing request. In general, if you have a running system you don't wish to disturb then `refresh=wait_for` is a smaller modification. +[float] === `refresh=wait_for` Can Force a Refresh If a `refresh=wait_for` request comes in when there are already @@ -74,6 +76,7 @@ contain `"forced_refresh": true`. Bulk requests only take up one slot on each shard that they touch no matter how many times they modify the shard. +[float] === Examples These will create a document and immediately refresh the index so it is visible: diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index 3311049c699..bb2a06c0234 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -375,10 +375,11 @@ parameter which causes just the shard that received the new data to be indexed. If the request contains `wait_for_completion=false` then Elasticsearch will perform some preflight checks, launch the request, and then return a `task` -which can be used with <> to cancel or get -the status of the task. For now, once the request is finished the task is gone -and the only place to look for the ultimate result of the task is in the -Elasticsearch log file. This will be fixed soon. +which can be used with <> +to cancel or get the status of the task. Elasticsearch will also create a +record of this task as a document at `.tasks/task/${taskId}`. This is yours +to keep or remove as you see fit. When you are done with it, delete it so +Elasticsearch can reclaim the space it uses. `consistency` controls how many copies of a shard must respond to each write request. `timeout` controls how long each write request waits for unavailable @@ -457,7 +458,7 @@ from aborting the operation. [[docs-reindex-task-api]] === Works with the Task API -While Reindex is running you can fetch their status using the +You can fetch the status of all running reindex requests with the <>: [source,js] @@ -515,6 +516,22 @@ of operations that the reindex expects to perform. You can estimate the progress by adding the `updated`, `created`, and `deleted` fields. The request will finish when their sum is equal to the `total` field. +With the task id you can look up the task directly: + +[source,js] +-------------------------------------------------- +GET /_tasks/taskId:1 +-------------------------------------------------- +// CONSOLE +// TEST[catch:missing] + +The advantage of this API is that it integrates with `wait_for_completion=false` +to transparently return the status of completed tasks. If the task is completed +and `wait_for_completion=false` was set on it them it'll come back with a +`results` or an `error` field. The cost of this feature is the document that +`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to +you to delete that document. + [float] [[docs-reindex-cancel-task-api]] diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index f3a147e36c1..6d6bfe64ecf 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -205,10 +205,11 @@ parameter which causes just the shard that received the new data to be indexed. If the request contains `wait_for_completion=false` then Elasticsearch will perform some preflight checks, launch the request, and then return a `task` -which can be used with <> to cancel -or get the status of the task. For now, once the request is finished the task -is gone and the only place to look for the ultimate result of the task is in -the Elasticsearch log file. This will be fixed soon. +which can be used with <> +to cancel or get the status of the task. Elasticsearch will also create a +record of this task as a document at `.tasks/task/${taskId}`. This is yours +to keep or remove as you see fit. When you are done with it, delete it so +Elasticsearch can reclaim the space it uses. `consistency` controls how many copies of a shard must respond to each write request. `timeout` controls how long each write request waits for unavailable @@ -283,7 +284,7 @@ from aborting the operation. [[docs-update-by-query-task-api]] === Works with the Task API -While Update By Query is running you can fetch their status using the +You can fetch the status of all running update-by-query requests with the <>: [source,js] @@ -341,6 +342,22 @@ of operations that the reindex expects to perform. You can estimate the progress by adding the `updated`, `created`, and `deleted` fields. The request will finish when their sum is equal to the `total` field. +With the task id you can look up the task directly: + +[source,js] +-------------------------------------------------- +GET /_tasks/taskId:1 +-------------------------------------------------- +// CONSOLE +// TEST[catch:missing] + +The advantage of this API is that it integrates with `wait_for_completion=false` +to transparently return the status of completed tasks. If the task is completed +and `wait_for_completion=false` was set on it them it'll come back with a +`results` or an `error` field. The cost of this feature is the document that +`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to +you to delete that document. + [float] [[docs-update-by-query-cancel-task-api]] diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index 8173b98d505..f1250ddf6c1 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -73,10 +73,11 @@ This similarity has the following options: [horizontal] `k1`:: Controls non-linear term frequency normalization - (saturation). + (saturation). The default value is `1.2`. `b`:: Controls to what degree document length normalizes tf values. + The default value is `0.75`. `discount_overlaps`:: Determines whether overlap tokens (Tokens with diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index ec641383beb..ce50f27451e 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -822,7 +822,7 @@ This is because the date is being rounded by month. | Name | Required | Default | Description | `field` | yes | - | The field to get the date or timestamp from. | `index_name_prefix` | no | - | A prefix of the index name to be prepended before the printed date. -| `date_rounding` | yes | - | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `m` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second). +| `date_rounding` | yes | - | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `M` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second). | `date_formats ` | no | yyyy-MM-dd'T'HH:mm:ss.SSSZ | An array of the expected date formats for parsing dates / timestamps in the document being preprocessed. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N. | `timezone` | no | UTC | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names. | `locale` | no | ENGLISH | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days. diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc index d90e429732b..d9227e91385 100644 --- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc +++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc @@ -20,3 +20,9 @@ Now that Elasticsearch supports `ipv6`, `ip` addresses are encoded in the index using a binary representation rather than a numeric representation. As a consequence, the output of `ip_range` aggregations does not give numeric values for `from` and `to` anymore. + +==== `size: 0` on Terms, Significant Terms and Geohash Grid Aggregations + +`size: 0` is no longer valid for the terms, significant terms and geohash grid +aggregations. Instead a size should be explicitly specified with a number greater +than zero. diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc index 62384afa60a..c20df657504 100644 --- a/docs/reference/migration/migrate_5_0/search.asciidoc +++ b/docs/reference/migration/migrate_5_0/search.asciidoc @@ -185,3 +185,10 @@ This is now consistent for source filtering on other places in the search API. In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to `description`. These changes have been made so the response format is more friendly to supporting other types of profiling in the future. + +==== Search prefernce + +The <> `_prefer_node` has +been superseded by `_prefer_nodes`. By specifying a single node, +`_prefer_nodes` provides the same functionality as `_prefer_node` but +also supports specifying multiple nodes. diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 2d8106e981c..d000805e5e9 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -328,7 +328,7 @@ By default, all indices in the snapshot are restored, and the cluster state is as to allow the global cluster state from being restored by using `indices` and `include_global_state` options in the restore request body. The list of indices supports <>. The `rename_pattern` -and `rename_replacement` options can be also used to rename index on restore +and `rename_replacement` options can be also used to rename indices on restore using regular expression that supports referencing the original text as explained http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]. diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index 699a9b37123..7ecc0f6554c 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -71,6 +71,10 @@ against the `test2` index. The `search_type` can be set in a similar manner to globally apply to all search requests. +The msearch's `max_concurrent_searches` request parameter can be used to control +the maximum number of concurrent searches the multi search api will execute. +This default is based on the number of data nodes and the default search thread pool size. + [float] [[msearch-security]] === Security diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 48b9de5ce60..25820d04800 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -324,8 +324,8 @@ Looking at the previous example: -------------------------------------------------- We see a single collector named `SimpleTopScoreDocCollector`. This is the default "scoring and sorting" Collector -used by Elasticsearch. The `"reason"` field attempts to give an plain english description of the class name. The -`"time` is similar to the time in the Query tree: a wall-clock time inclusive of all children. Similarly, `children` lists +used by Elasticsearch. The `"reason"` field attempts to give a plain english description of the class name. The +`"time"` is similar to the time in the Query tree: a wall-clock time inclusive of all children. Similarly, `children` lists all sub-collectors. It should be noted that Collector times are **independent** from the Query times. They are calculated, combined diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 8207c6577fe..958320ea110 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -55,7 +55,7 @@ And here is a sample response: `from`:: - The starting from index of the hits to return. Defaults to `0`. + To retrieve hits from a certain offset. Defaults to `0`. `size`:: diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc index 3d6c6b40cb9..761fdeaa369 100644 --- a/docs/reference/search/request/preference.asciidoc +++ b/docs/reference/search/request/preference.asciidoc @@ -31,9 +31,9 @@ The `preference` is a query string parameter which can be set to: Restricts the search to execute only on a node with the provided node id (`xyz` in this case). -`_prefer_node:xyz`:: - Prefers execution on the node with the provided - node id (`xyz` in this case) if applicable. +`_prefer_nodes:abc,xyz`:: + Prefers execution on the nodes with the provided + node ids (`abc` or `xyz` in this case) if applicable. `_shards:2,3`:: Restricts the operation to the specified shards. (`2` diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc index 9f9558aa979..a082bf3ba4c 100644 --- a/docs/reference/search/request/scroll.asciidoc +++ b/docs/reference/search/request/scroll.asciidoc @@ -263,4 +263,7 @@ curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d ' ' -------------------------------------------------- -For append only time-based indices, the `timestamp` field can be used safely. \ No newline at end of file +For append only time-based indices, the `timestamp` field can be used safely. + +NOTE: By default the maximum number of slices allowed per scroll is limited to 1024. +You can update the `index.max_slices_per_scroll` index setting to bypass this limit. \ No newline at end of file diff --git a/docs/src/test/cluster/config/KeywordTokenizer.rbbi b/docs/src/test/cluster/config/KeywordTokenizer.rbbi new file mode 100644 index 00000000000..b08617c8991 --- /dev/null +++ b/docs/src/test/cluster/config/KeywordTokenizer.rbbi @@ -0,0 +1 @@ +.+ {200}; diff --git a/docs/src/test/cluster/config/scripts/my_script.js b/docs/src/test/cluster/config/scripts/my_script.js new file mode 100644 index 00000000000..cc985c18a94 --- /dev/null +++ b/docs/src/test/cluster/config/scripts/my_script.js @@ -0,0 +1 @@ +doc["num"].value * factor diff --git a/docs/src/test/cluster/config/scripts/my_script.py b/docs/src/test/cluster/config/scripts/my_script.py new file mode 100644 index 00000000000..cc985c18a94 --- /dev/null +++ b/docs/src/test/cluster/config/scripts/my_script.py @@ -0,0 +1 @@ +doc["num"].value * factor diff --git a/docs/src/test/cluster/config/userdict_ja.txt b/docs/src/test/cluster/config/userdict_ja.txt new file mode 100644 index 00000000000..7fe69d2b8d8 --- /dev/null +++ b/docs/src/test/cluster/config/userdict_ja.txt @@ -0,0 +1 @@ +東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞 diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1 deleted file mode 100644 index 7b3f5a1cef9..00000000000 --- a/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2b76056dbd40fb51dc5e8ef71e1919ad23e635a1 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..271088e86c9 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +e5a4b673918f448006c0531799706abebe9a1db0 \ No newline at end of file diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java index ced13b8e53a..6ed77fffcc8 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java @@ -63,6 +63,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -235,20 +236,12 @@ public class DoubleTermsTests extends AbstractTermsTestCase { // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard public void testSizeIsZero() { - SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") - .addAggregation(terms("terms") - .field(SINGLE_VALUED_FIELD_NAME) - .minDocCount(randomInt(1)) - .size(0) - .collectMode(randomFrom(SubAggCollectionMode.values()))) - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(100)); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client() + .prepareSearch("idx").setTypes("high_card_type").addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME) + .minDocCount(randomInt(1)).size(0).collectMode(randomFrom(SubAggCollectionMode.values()))) + .execute().actionGet()); + assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } public void testSingleValueField() throws Exception { @@ -594,7 +587,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) - .size(randomInt(5)) + .size(randomIntBetween(1, 5)) .collectMode(randomFrom(SubAggCollectionMode.values()))) .execute().actionGet(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java index 65f288113ab..fa049429a3a 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java @@ -61,6 +61,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -237,20 +238,15 @@ public class LongTermsTests extends AbstractTermsTestCase { // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard public void testSizeIsZero() { - SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type") + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> client().prepareSearch("idx").setTypes("high_card_type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())) .minDocCount(randomInt(1)) .size(0)) - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(100)); + .execute().actionGet()); + assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } public void testSingleValueField() throws Exception { @@ -596,7 +592,7 @@ public class LongTermsTests extends AbstractTermsTestCase { SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type") .addAggregation(terms("terms") .field(SINGLE_VALUED_FIELD_NAME) - .size(randomInt(5)) + .size(randomIntBetween(1, 5)) .collectMode(randomFrom(SubAggCollectionMode.values()))) .execute().actionGet(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java index 11f0e4c191a..78d95edecff 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java @@ -23,6 +23,7 @@ package org.elasticsearch.messy.tests; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; @@ -41,7 +42,6 @@ import java.util.Collections; import java.util.List; import java.util.Random; -import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -218,7 +218,7 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L)); for (int i = 0; i < 10; i++) { - assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, TOLERANCE)); + assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, GeoUtils.TOLERANCE)); } } diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java index deeeb2911b6..b098db154da 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java @@ -70,6 +70,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -202,20 +203,14 @@ public class StringTermsTests extends AbstractTermsTestCase { // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard public void testSizeIsZero() { final int minDocCount = randomInt(1); - SearchResponse response = client() + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> client() .prepareSearch("idx") .setTypes("high_card_type") .addAggregation( terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values())).minDocCount(minDocCount).size(0)).execute() - .actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - assertThat(terms, notNullValue()); - assertThat(terms.getName(), equalTo("terms")); - assertThat(terms.getBuckets().size(), equalTo(minDocCount == 0 ? 105 : 100)); // 105 because of the other type + .actionGet()); + assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]")); } public void testSingleValueField() throws Exception { @@ -760,7 +755,7 @@ public class StringTermsTests extends AbstractTermsTestCase { .prepareSearch("idx_unmapped") .setTypes("type") .addAggregation( - terms("terms").executionHint(randomExecutionHint()).size(randomInt(5)).field(SINGLE_VALUED_FIELD_NAME) + terms("terms").executionHint(randomExecutionHint()).size(randomIntBetween(1, 5)).field(SINGLE_VALUED_FIELD_NAME) .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet(); assertSearchResponse(response); diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 9f617c4e436..25a348927d7 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -26,10 +26,7 @@ esplugin { dependencies { compile 'org.antlr:antlr4-runtime:4.5.1-1' - compile 'org.ow2.asm:asm:5.0.4' - compile 'org.ow2.asm:asm-commons:5.0.4' - compile 'org.ow2.asm:asm-tree:5.0.4' - testCompile 'org.ow2.asm:asm-util:5.0.4' + compile 'org.ow2.asm:asm-debug-all:5.1' } dependencyLicenses { diff --git a/modules/lang-painless/licenses/asm-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-5.0.4.jar.sha1 deleted file mode 100644 index 9223dba380f..00000000000 --- a/modules/lang-painless/licenses/asm-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0da08b8cce7bbf903602a25a3a163ae252435795 diff --git a/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1 deleted file mode 100644 index 94fe0cd92c9..00000000000 --- a/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a556786086c23cd689a0328f8519db93821c04c diff --git a/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1 b/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1 new file mode 100644 index 00000000000..926910cbc68 --- /dev/null +++ b/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1 @@ -0,0 +1 @@ +dde860d586960d55a87cd52c4fc1f5059c89cbc6 \ No newline at end of file diff --git a/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1 deleted file mode 100644 index 5822a485a61..00000000000 --- a/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -396ce0c07ba2b481f25a70195c7c94922f0d1b0b \ No newline at end of file diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4 index 4ca5778ad3c..fac36030120 100644 --- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 +++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4 @@ -51,11 +51,12 @@ NEW: 'new'; TRY: 'try'; CATCH: 'catch'; THROW: 'throw'; +THIS: 'this'; BOOLNOT: '!'; BWNOT: '~'; MUL: '*'; -DIV: '/'; +DIV: '/' { false == SlashStrategy.slashIsRegex(_factory) }?; REM: '%'; ADD: '+'; SUB: '-'; @@ -78,6 +79,7 @@ BOOLOR: '||'; COND: '?'; COLON: ':'; REF: '::'; +ARROW: '->'; INCR: '++'; DECR: '--'; @@ -100,6 +102,7 @@ INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?; DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?; STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' ); +REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' { SlashStrategy.slashIsRegex(_factory) }?; TRUE: 'true'; FALSE: 'false'; diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens index 567fb8fdddd..b6df2d97a51 100644 --- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens +++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens @@ -21,58 +21,61 @@ NEW=20 TRY=21 CATCH=22 THROW=23 -BOOLNOT=24 -BWNOT=25 -MUL=26 -DIV=27 -REM=28 -ADD=29 -SUB=30 -LSH=31 -RSH=32 -USH=33 -LT=34 -LTE=35 -GT=36 -GTE=37 -EQ=38 -EQR=39 -NE=40 -NER=41 -BWAND=42 -XOR=43 -BWOR=44 -BOOLAND=45 -BOOLOR=46 -COND=47 -COLON=48 -REF=49 -INCR=50 -DECR=51 -ASSIGN=52 -AADD=53 -ASUB=54 -AMUL=55 -ADIV=56 -AREM=57 -AAND=58 -AXOR=59 -AOR=60 -ALSH=61 -ARSH=62 -AUSH=63 -OCTAL=64 -HEX=65 -INTEGER=66 -DECIMAL=67 -STRING=68 -TRUE=69 -FALSE=70 -NULL=71 -TYPE=72 -ID=73 -DOTINTEGER=74 -DOTID=75 +THIS=24 +BOOLNOT=25 +BWNOT=26 +MUL=27 +DIV=28 +REM=29 +ADD=30 +SUB=31 +LSH=32 +RSH=33 +USH=34 +LT=35 +LTE=36 +GT=37 +GTE=38 +EQ=39 +EQR=40 +NE=41 +NER=42 +BWAND=43 +XOR=44 +BWOR=45 +BOOLAND=46 +BOOLOR=47 +COND=48 +COLON=49 +REF=50 +ARROW=51 +INCR=52 +DECR=53 +ASSIGN=54 +AADD=55 +ASUB=56 +AMUL=57 +ADIV=58 +AREM=59 +AAND=60 +AXOR=61 +AOR=62 +ALSH=63 +ARSH=64 +AUSH=65 +OCTAL=66 +HEX=67 +INTEGER=68 +DECIMAL=69 +STRING=70 +REGEX=71 +TRUE=72 +FALSE=73 +NULL=74 +TYPE=75 +ID=76 +DOTINTEGER=77 +DOTID=78 '{'=3 '}'=4 '['=5 @@ -94,46 +97,48 @@ DOTID=75 'try'=21 'catch'=22 'throw'=23 -'!'=24 -'~'=25 -'*'=26 -'/'=27 -'%'=28 -'+'=29 -'-'=30 -'<<'=31 -'>>'=32 -'>>>'=33 -'<'=34 -'<='=35 -'>'=36 -'>='=37 -'=='=38 -'==='=39 -'!='=40 -'!=='=41 -'&'=42 -'^'=43 -'|'=44 -'&&'=45 -'||'=46 -'?'=47 -':'=48 -'::'=49 -'++'=50 -'--'=51 -'='=52 -'+='=53 -'-='=54 -'*='=55 -'/='=56 -'%='=57 -'&='=58 -'^='=59 -'|='=60 -'<<='=61 -'>>='=62 -'>>>='=63 -'true'=69 -'false'=70 -'null'=71 +'this'=24 +'!'=25 +'~'=26 +'*'=27 +'/'=28 +'%'=29 +'+'=30 +'-'=31 +'<<'=32 +'>>'=33 +'>>>'=34 +'<'=35 +'<='=36 +'>'=37 +'>='=38 +'=='=39 +'==='=40 +'!='=41 +'!=='=42 +'&'=43 +'^'=44 +'|'=45 +'&&'=46 +'||'=47 +'?'=48 +':'=49 +'::'=50 +'->'=51 +'++'=52 +'--'=53 +'='=54 +'+='=55 +'-='=56 +'*='=57 +'/='=58 +'%='=59 +'&='=60 +'^='=61 +'|='=62 +'<<='=63 +'>>='=64 +'>>>='=65 +'true'=72 +'false'=73 +'null'=74 diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4 index 141ff25e318..cc2ca89047f 100644 --- a/modules/lang-painless/src/main/antlr/PainlessParser.g4 +++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4 @@ -22,7 +22,15 @@ parser grammar PainlessParser; options { tokenVocab=PainlessLexer; } source - : statement* EOF + : function* statement* EOF + ; + +function + : decltype ID parameters block + ; + +parameters + : LP ( decltype ID ( COMMA decltype ID )* )? RP ; // Note we use a predicate on the if/else case here to prevent the @@ -73,10 +81,6 @@ decltype : TYPE (LBRACE RBRACE)* ; -funcref - : TYPE REF ( ID | NEW ) - ; - declvar : ID ( ASSIGN expression )? ; @@ -88,6 +92,17 @@ trap delimiter : SEMICOLON | EOF + // RBRACK is a delimiter but we don't consume it because it is only valid + // in places where RBRACK can follow the statement. It is simpler to not + // consume it here then it is to consume it here. Unfortunately, they + // obvious syntax to do this `| { _input.LA(1) == RBRACK }?` generates an + // amazingly intense `adaptivePredict` call that doesn't actually work + // and builds a serious DFA. Huge. So instead we use standard ANTLR syntax + // to consume the token and then undo the consumption. This looks hairy but + // it is better than the alternatives. + | { int mark = _input.mark(); int index = _input.index(); } + RBRACK + { _input.seek(index); _input.release(mark); } ; // Note we return the boolean s. This is returned as true @@ -142,7 +157,9 @@ primary[boolean c] returns [boolean s = true] : { !$c }? LP e = expression RP { $s = $e.s; } # exprprec | { $c }? LP unary[true] RP # chainprec | STRING # string + | REGEX # regex | ID # variable + | ID arguments # calllocal | NEW TYPE arguments # newobject ; @@ -166,5 +183,43 @@ arguments argument : expression + | lambda | funcref ; + +lambda + : ( lamtype | LP ( lamtype ( COMMA lamtype )* )? RP ) ARROW block + ; + +lamtype + : decltype? ID + ; + +funcref + : classFuncref + | constructorFuncref + | capturingFuncref + | localFuncref + ; + +// reference to a static or instance method, e.g. ArrayList::size or Integer::compare +classFuncref + : TYPE REF ID + ; + +// reference to a constructor, e.g. ArrayList::new +// currently limited to simple non-array types +constructorFuncref + : decltype REF NEW + ; + +// reference to an instance method, e.g. object::toString +// currently limited to capture of a simple variable (id). +capturingFuncref + : ID REF ID + ; + +// reference to a local function, e.g. this::myfunc +localFuncref + : THIS REF ID + ; diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens index 567fb8fdddd..b6df2d97a51 100644 --- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens +++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens @@ -21,58 +21,61 @@ NEW=20 TRY=21 CATCH=22 THROW=23 -BOOLNOT=24 -BWNOT=25 -MUL=26 -DIV=27 -REM=28 -ADD=29 -SUB=30 -LSH=31 -RSH=32 -USH=33 -LT=34 -LTE=35 -GT=36 -GTE=37 -EQ=38 -EQR=39 -NE=40 -NER=41 -BWAND=42 -XOR=43 -BWOR=44 -BOOLAND=45 -BOOLOR=46 -COND=47 -COLON=48 -REF=49 -INCR=50 -DECR=51 -ASSIGN=52 -AADD=53 -ASUB=54 -AMUL=55 -ADIV=56 -AREM=57 -AAND=58 -AXOR=59 -AOR=60 -ALSH=61 -ARSH=62 -AUSH=63 -OCTAL=64 -HEX=65 -INTEGER=66 -DECIMAL=67 -STRING=68 -TRUE=69 -FALSE=70 -NULL=71 -TYPE=72 -ID=73 -DOTINTEGER=74 -DOTID=75 +THIS=24 +BOOLNOT=25 +BWNOT=26 +MUL=27 +DIV=28 +REM=29 +ADD=30 +SUB=31 +LSH=32 +RSH=33 +USH=34 +LT=35 +LTE=36 +GT=37 +GTE=38 +EQ=39 +EQR=40 +NE=41 +NER=42 +BWAND=43 +XOR=44 +BWOR=45 +BOOLAND=46 +BOOLOR=47 +COND=48 +COLON=49 +REF=50 +ARROW=51 +INCR=52 +DECR=53 +ASSIGN=54 +AADD=55 +ASUB=56 +AMUL=57 +ADIV=58 +AREM=59 +AAND=60 +AXOR=61 +AOR=62 +ALSH=63 +ARSH=64 +AUSH=65 +OCTAL=66 +HEX=67 +INTEGER=68 +DECIMAL=69 +STRING=70 +REGEX=71 +TRUE=72 +FALSE=73 +NULL=74 +TYPE=75 +ID=76 +DOTINTEGER=77 +DOTID=78 '{'=3 '}'=4 '['=5 @@ -94,46 +97,48 @@ DOTID=75 'try'=21 'catch'=22 'throw'=23 -'!'=24 -'~'=25 -'*'=26 -'/'=27 -'%'=28 -'+'=29 -'-'=30 -'<<'=31 -'>>'=32 -'>>>'=33 -'<'=34 -'<='=35 -'>'=36 -'>='=37 -'=='=38 -'==='=39 -'!='=40 -'!=='=41 -'&'=42 -'^'=43 -'|'=44 -'&&'=45 -'||'=46 -'?'=47 -':'=48 -'::'=49 -'++'=50 -'--'=51 -'='=52 -'+='=53 -'-='=54 -'*='=55 -'/='=56 -'%='=57 -'&='=58 -'^='=59 -'|='=60 -'<<='=61 -'>>='=62 -'>>>='=63 -'true'=69 -'false'=70 -'null'=71 +'this'=24 +'!'=25 +'~'=26 +'*'=27 +'/'=28 +'%'=29 +'+'=30 +'-'=31 +'<<'=32 +'>>'=33 +'>>>'=34 +'<'=35 +'<='=36 +'>'=37 +'>='=38 +'=='=39 +'==='=40 +'!='=41 +'!=='=42 +'&'=43 +'^'=44 +'|'=45 +'&&'=46 +'||'=47 +'?'=48 +':'=49 +'::'=50 +'->'=51 +'++'=52 +'--'=53 +'='=54 +'+='=55 +'-='=56 +'*='=57 +'/='=58 +'%='=59 +'&='=60 +'^='=61 +'|='=62 +'<<='=63 +'>>='=64 +'>>>='=65 +'true'=72 +'false'=73 +'null'=74 diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java index e724fa9e810..a37e4a4b0aa 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java @@ -40,6 +40,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case BOOL_OBJ: if (internal) return new Cast(actual, actual, explicit, false, false, false, true); @@ -62,6 +66,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case BYTE_OBJ: if (internal) @@ -117,6 +125,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case SHORT_OBJ: if (internal) @@ -172,6 +184,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case CHAR_OBJ: if (internal) @@ -229,6 +245,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case INT_OBJ: if (internal) @@ -284,6 +304,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case LONG_OBJ: if (internal) @@ -339,6 +363,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case FLOAT_OBJ: if (internal) @@ -392,6 +420,10 @@ public final class AnalyzerCaster { case DEF: return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false); case OBJECT: + if (Definition.OBJECT_TYPE.equals(expected) && internal) + return new Cast(actual, actual, explicit, false, false, false, true); + + break; case NUMBER: case DOUBLE_OBJ: if (internal) @@ -432,6 +464,45 @@ public final class AnalyzerCaster { break; case OBJECT: + if (Definition.OBJECT_TYPE.equals(actual)) + switch (expected.sort) { + case BYTE: + if (internal && explicit) + return new Cast(actual, Definition.BYTE_OBJ_TYPE, true, false, true, false, false); + + break; + case SHORT: + if (internal && explicit) + return new Cast(actual, Definition.SHORT_OBJ_TYPE, true, false, true, false, false); + + break; + case CHAR: + if (internal && explicit) + return new Cast(actual, Definition.CHAR_OBJ_TYPE, true, false, true, false, false); + + break; + case INT: + if (internal && explicit) + return new Cast(actual, Definition.INT_OBJ_TYPE, true, false, true, false, false); + + break; + case LONG: + if (internal && explicit) + return new Cast(actual, Definition.LONG_OBJ_TYPE, true, false, true, false, false); + + break; + case FLOAT: + if (internal && explicit) + return new Cast(actual, Definition.FLOAT_OBJ_TYPE, true, false, true, false, false); + + break; + case DOUBLE: + if (internal && explicit) + return new Cast(actual, Definition.DOUBLE_OBJ_TYPE, true, false, true, false, false); + + break; + } + break; case NUMBER: switch (expected.sort) { case BYTE: diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index d0a17f64a48..8fa9f5d583e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -20,7 +20,6 @@ package org.elasticsearch.painless; import org.elasticsearch.bootstrap.BootstrapInfo; -import org.elasticsearch.painless.Variables.Reserved; import org.elasticsearch.painless.antlr.Walker; import org.elasticsearch.painless.node.SSource; @@ -36,8 +35,7 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_NAME; /** * The Compiler is the entry point for generating a Painless script. The compiler will receive a Painless * tree based on the type of input passed in (currently only ANTLR). Two passes will then be run over the tree, - * one for analysis using the {@link Analyzer} and another to generate the actual byte code using ASM in - * the {@link Writer}. + * one for analysis and another to generate the actual byte code using ASM using the root of the tree {@link SSource}. */ final class Compiler { @@ -100,18 +98,17 @@ final class Compiler { " plugin if a script longer than this length is a requirement."); } - Reserved reserved = new Reserved(); - SSource root = Walker.buildPainlessTree(name, source, reserved, settings); - Variables variables = Analyzer.analyze(reserved, root); - BitSet expressions = new BitSet(source.length()); - byte[] bytes = Writer.write(settings, name, source, variables, root, expressions); + SSource root = Walker.buildPainlessTree(name, source, settings); + + root.analyze(); + root.write(); try { - Class clazz = loader.define(CLASS_NAME, bytes); + Class clazz = loader.define(CLASS_NAME, root.getBytes()); java.lang.reflect.Constructor constructor = clazz.getConstructor(String.class, String.class, BitSet.class); - return constructor.newInstance(name, source, expressions); + return constructor.newInstance(name, source, root.getExpressions()); } catch (Exception exception) { // Catch everything to let the user know this is something caused internally. throw new IllegalStateException("An internal error occurred attempting to define the script [" + name + "].", exception); } @@ -130,11 +127,12 @@ final class Compiler { " plugin if a script longer than this length is a requirement."); } - Reserved reserved = new Reserved(); - SSource root = Walker.buildPainlessTree(name, source, reserved, settings); - Variables variables = Analyzer.analyze(reserved, root); + SSource root = Walker.buildPainlessTree(name, source, settings); - return Writer.write(settings, name, source, variables, root, new BitSet(source.length())); + root.analyze(); + root.write(); + + return root.getBytes(); } /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 1edaddea2c1..158086ab429 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -23,7 +23,6 @@ import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.RuntimeClass; import java.lang.invoke.CallSite; -import java.lang.invoke.LambdaConversionException; import java.lang.invoke.LambdaMetafactory; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; @@ -212,39 +211,133 @@ public final class Def { * until it finds a matching whitelisted method. If one is not found, it throws an exception. * Otherwise it returns a handle to the matching method. *

+ * @param lookup caller's lookup + * @param callSiteType callsite's type * @param receiverClass Class of the object to invoke the method on. * @param name Name of the method. * @param args args passed to callsite * @param recipe bitset marking functional parameters * @return pointer to matching method to invoke. never returns null. - * @throws LambdaConversionException if a method reference cannot be converted to an functional interface * @throws IllegalArgumentException if no matching whitelisted method was found. + * @throws Throwable if a method reference cannot be converted to an functional interface */ - static MethodHandle lookupMethod(Lookup lookup, Class receiverClass, String name, - Object args[], long recipe) throws LambdaConversionException { - Method method = lookupMethodInternal(receiverClass, name, args.length - 1); + static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, + Class receiverClass, String name, Object args[], long recipe) throws Throwable { + // simple case: no lambdas + if (recipe == 0) { + return lookupMethodInternal(receiverClass, name, args.length - 1).handle; + } + + // otherwise: first we have to compute the "real" arity. This is because we have extra arguments: + // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i). + int arity = args.length - 1; + for (int i = 0; i < args.length; i++) { + if ((recipe & (1L << (i - 1))) != 0) { + String signature = (String) args[i]; + int numCaptures = Integer.parseInt(signature.substring(signature.indexOf(',')+1)); + arity -= numCaptures; + } + } + + // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters). + // based on these we can finally link any remaining lambdas that were deferred. + Method method = lookupMethodInternal(receiverClass, name, arity); MethodHandle handle = method.handle; - if (recipe != 0) { - MethodHandle filters[] = new MethodHandle[args.length]; - for (int i = 0; i < args.length; i++) { - // its a functional reference, replace the argument with an impl - if ((recipe & (1L << (i - 1))) != 0) { - filters[i] = lookupReference(lookup, method.arguments.get(i - 1), (String) args[i]); + int replaced = 0; + for (int i = 1; i < args.length; i++) { + // its a functional reference, replace the argument with an impl + if ((recipe & (1L << (i - 1))) != 0) { + // decode signature of form 'type.call,2' + String signature = (String) args[i]; + int separator = signature.indexOf('.'); + int separator2 = signature.indexOf(','); + String type = signature.substring(1, separator); + String call = signature.substring(separator+1, separator2); + int numCaptures = Integer.parseInt(signature.substring(separator2+1)); + Class captures[] = new Class[numCaptures]; + for (int capture = 0; capture < captures.length; capture++) { + captures[capture] = callSiteType.parameterType(i + 1 + capture); } + MethodHandle filter; + Definition.Type interfaceType = method.arguments.get(i - 1 - replaced); + if (signature.charAt(0) == 'S') { + // the implementation is strongly typed, now that we know the interface type, + // we have everything. + filter = lookupReferenceInternal(lookup, + interfaceType, + type, + call, + captures); + } else if (signature.charAt(0) == 'D') { + // the interface type is now known, but we need to get the implementation. + // this is dynamically based on the receiver type (and cached separately, underneath + // this cache). It won't blow up since we never nest here (just references) + MethodType nestedType = MethodType.methodType(interfaceType.clazz, captures); + CallSite nested = DefBootstrap.bootstrap(lookup, + call, + nestedType, + DefBootstrap.REFERENCE, + interfaceType.name); + filter = nested.dynamicInvoker(); + } else { + throw new AssertionError(); + } + // the filter now ignores the signature (placeholder) on the stack + filter = MethodHandles.dropArguments(filter, 0, String.class); + handle = MethodHandles.collectArguments(handle, i, filter); + i += numCaptures; + replaced += numCaptures; } - handle = MethodHandles.filterArguments(handle, 0, filters); } return handle; } - /** Returns a method handle to an implementation of clazz, given method reference signature - * @throws LambdaConversionException if a method reference cannot be converted to an functional interface + /** + * Returns an implementation of interfaceClass that calls receiverClass.name + *

+ * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known, + * so we simply need to lookup the matching implementation method based on receiver type. */ - private static MethodHandle lookupReference(Lookup lookup, Definition.Type clazz, String signature) throws LambdaConversionException { - int separator = signature.indexOf('.'); - FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1)); + static MethodHandle lookupReference(Lookup lookup, String interfaceClass, + Class receiverClass, String name) throws Throwable { + Definition.Type interfaceType = Definition.getType(interfaceClass); + Method interfaceMethod = interfaceType.struct.getFunctionalMethod(); + if (interfaceMethod == null) { + throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface"); + } + int arity = interfaceMethod.arguments.size(); + Method implMethod = lookupMethodInternal(receiverClass, name, arity); + return lookupReferenceInternal(lookup, interfaceType, implMethod.owner.name, implMethod.name, receiverClass); + } + + /** Returns a method handle to an implementation of clazz, given method reference signature. */ + private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Type clazz, String type, + String call, Class... captures) throws Throwable { + final FunctionRef ref; + if ("this".equals(type)) { + // user written method + Method interfaceMethod = clazz.struct.getFunctionalMethod(); + if (interfaceMethod == null) { + throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + + "to [" + clazz.name + "], not a functional interface"); + } + int arity = interfaceMethod.arguments.size() + captures.length; + final MethodHandle handle; + try { + MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(), + getUserFunctionHandleFieldName(call, arity), + MethodHandle.class); + handle = (MethodHandle) accessor.invokeExact(); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments."); + } + ref = new FunctionRef(clazz, interfaceMethod, handle, captures); + } else { + // whitelist lookup + ref = new FunctionRef(clazz, type, call, captures); + } final CallSite callSite; if (ref.needsBridges()) { callSite = LambdaMetafactory.altMetafactory(lookup, @@ -265,11 +358,13 @@ public final class Def { ref.samMethodType, 0); } - // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup. - MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz)); - return MethodHandles.dropArguments(factory, 0, Object.class); + return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures)); } + /** gets the field name used to lookup up the MethodHandle for a function. */ + public static String getUserFunctionHandleFieldName(String name, int arity) { + return "handle$" + name + "$" + arity; + } /** * Looks up handle for a dynamic field getter (field load) @@ -560,625 +655,6 @@ public final class Def { } } - // NOTE: Below methods are not cached, instead invoked directly because they are performant. - // We also check for Long values first when possible since the type is more - // likely to be a Long than a Float. - - public static Object not(final Object unary) { - if (unary instanceof Double || unary instanceof Long || unary instanceof Float) { - return ~((Number)unary).longValue(); - } else if (unary instanceof Number) { - return ~((Number)unary).intValue(); - } else if (unary instanceof Character) { - return ~(int)(char)unary; - } - - throw new ClassCastException("Cannot apply [~] operation to type " + - "[" + unary.getClass().getCanonicalName() + "]."); - } - - public static Object neg(final Object unary) { - if (unary instanceof Double) { - return -(double)unary; - } else if (unary instanceof Float) { - return -(float)unary; - } else if (unary instanceof Long) { - return -(long)unary; - } else if (unary instanceof Number) { - return -((Number)unary).intValue(); - } else if (unary instanceof Character) { - return -(char)unary; - } - - throw new ClassCastException("Cannot apply [-] operation to type " + - "[" + unary.getClass().getCanonicalName() + "]."); - } - - public static Object mul(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() * ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() * ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() * ((Number)right).longValue(); - } else { - return ((Number)left).intValue() * ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() * (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() * (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() * (char)right; - } else { - return ((Number)left).intValue() * (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left * ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left * ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left * ((Number)right).floatValue(); - } else { - return (char)left * ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left * (char)right; - } - } - - throw new ClassCastException("Cannot apply [*] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object div(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() / ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() / ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() / ((Number)right).longValue(); - } else { - return ((Number)left).intValue() / ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() / (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() / (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() / (char)right; - } else { - return ((Number)left).intValue() / (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left / ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left / ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left / ((Number)right).floatValue(); - } else { - return (char)left / ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left / (char)right; - } - } - - throw new ClassCastException("Cannot apply [/] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object rem(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() % ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() % ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() % ((Number)right).longValue(); - } else { - return ((Number)left).intValue() % ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() % (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() % (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() % (char)right; - } else { - return ((Number)left).intValue() % (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left % ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left % ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left % ((Number)right).floatValue(); - } else { - return (char)left % ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left % (char)right; - } - } - - throw new ClassCastException("Cannot apply [%] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object add(final Object left, final Object right) { - if (left instanceof String || right instanceof String) { - return "" + left + right; - } else if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() + ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() + ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() + ((Number)right).longValue(); - } else { - return ((Number)left).intValue() + ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() + (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() + (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() + (char)right; - } else { - return ((Number)left).intValue() + (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left + ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left + ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left + ((Number)right).floatValue(); - } else { - return (char)left + ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left + (char)right; - } - } - - throw new ClassCastException("Cannot apply [+] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object sub(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() - ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() - ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() - ((Number)right).longValue(); - } else { - return ((Number)left).intValue() - ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() - (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() - (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() - (char)right; - } else { - return ((Number)left).intValue() - (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left - ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left - ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left - ((Number)right).floatValue(); - } else { - return (char)left - ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left - (char)right; - } - } - - throw new ClassCastException("Cannot apply [-] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object lsh(final Object left, final int right) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() << right; - } else if (left instanceof Number) { - return ((Number)left).intValue() << right; - } else if (left instanceof Character) { - return (char)left << right; - } - - throw new ClassCastException("Cannot apply [<<] operation to types [" + left.getClass().getCanonicalName() + "] and [int]."); - } - - public static Object rsh(final Object left, final int right) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() >> right; - } else if (left instanceof Number) { - return ((Number)left).intValue() >> right; - } else if (left instanceof Character) { - return (char)left >> right; - } - - throw new ClassCastException("Cannot apply [>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int]."); - } - - public static Object ush(final Object left, final int right) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() >>> right; - } else if (left instanceof Number) { - return ((Number)left).intValue() >>> right; - } else if (left instanceof Character) { - return (char)left >>> right; - } - - throw new ClassCastException("Cannot apply [>>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int]."); - } - - public static Object and(final Object left, final Object right) { - if (left instanceof Boolean && right instanceof Boolean) { - return (boolean)left && (boolean)right; - } else if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double || - left instanceof Long || right instanceof Long || - left instanceof Float || right instanceof Float) { - return ((Number)left).longValue() & ((Number)right).longValue(); - } else { - return ((Number)left).intValue() & ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() & (char)right; - } else { - return ((Number)left).intValue() & (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double || right instanceof Long || right instanceof Float) { - return (char)left & ((Number)right).longValue(); - } else { - return (char)left & ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left & (char)right; - } - } - - throw new ClassCastException("Cannot apply [&] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object xor(final Object left, final Object right) { - if (left instanceof Boolean && right instanceof Boolean) { - return (boolean)left ^ (boolean)right; - } else if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double || - left instanceof Long || right instanceof Long || - left instanceof Float || right instanceof Float) { - return ((Number)left).longValue() ^ ((Number)right).longValue(); - } else { - return ((Number)left).intValue() ^ ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() ^ (char)right; - } else { - return ((Number)left).intValue() ^ (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double || right instanceof Long || right instanceof Float) { - return (char)left ^ ((Number)right).longValue(); - } else { - return (char)left ^ ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left ^ (char)right; - } - } - - throw new ClassCastException("Cannot apply [^] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static Object or(final Object left, final Object right) { - if (left instanceof Boolean && right instanceof Boolean) { - return (boolean)left || (boolean)right; - } else if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double || - left instanceof Long || right instanceof Long || - left instanceof Float || right instanceof Float) { - return ((Number)left).longValue() | ((Number)right).longValue(); - } else { - return ((Number)left).intValue() | ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double || left instanceof Long || left instanceof Float) { - return ((Number)left).longValue() | (char)right; - } else { - return ((Number)left).intValue() | (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double || right instanceof Long || right instanceof Float) { - return (char)left | ((Number)right).longValue(); - } else { - return (char)left | ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left | (char)right; - } - } - - throw new ClassCastException("Cannot apply [|] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static boolean eq(final Object left, final Object right) { - if (left != null && right != null) { - if (left instanceof Double) { - if (right instanceof Number) { - return (double)left == ((Number)right).doubleValue(); - } else if (right instanceof Character) { - return (double)left == (char)right; - } - } else if (right instanceof Double) { - if (left instanceof Number) { - return ((Number)left).doubleValue() == (double)right; - } else if (left instanceof Character) { - return (char)left == ((Number)right).doubleValue(); - } - } else if (left instanceof Float) { - if (right instanceof Number) { - return (float)left == ((Number)right).floatValue(); - } else if (right instanceof Character) { - return (float)left == (char)right; - } - } else if (right instanceof Float) { - if (left instanceof Number) { - return ((Number)left).floatValue() == (float)right; - } else if (left instanceof Character) { - return (char)left == ((Number)right).floatValue(); - } - } else if (left instanceof Long) { - if (right instanceof Number) { - return (long)left == ((Number)right).longValue(); - } else if (right instanceof Character) { - return (long)left == (char)right; - } - } else if (right instanceof Long) { - if (left instanceof Number) { - return ((Number)left).longValue() == (long)right; - } else if (left instanceof Character) { - return (char)left == ((Number)right).longValue(); - } - } else if (left instanceof Number) { - if (right instanceof Number) { - return ((Number)left).intValue() == ((Number)right).intValue(); - } else if (right instanceof Character) { - return ((Number)left).intValue() == (char)right; - } - } else if (right instanceof Number && left instanceof Character) { - return (char)left == ((Number)right).intValue(); - } else if (left instanceof Character && right instanceof Character) { - return (char)left == (char)right; - } - - return left.equals(right); - } - - return left == null && right == null; - } - - public static boolean lt(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() < ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() < ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() < ((Number)right).longValue(); - } else { - return ((Number)left).intValue() < ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() < (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() < (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() < (char)right; - } else { - return ((Number)left).intValue() < (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left < ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left < ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left < ((Number)right).floatValue(); - } else { - return (char)left < ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left < (char)right; - } - } - - throw new ClassCastException("Cannot apply [<] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static boolean lte(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() <= ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() <= ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() <= ((Number)right).longValue(); - } else { - return ((Number)left).intValue() <= ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() <= (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() <= (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() <= (char)right; - } else { - return ((Number)left).intValue() <= (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left <= ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left <= ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left <= ((Number)right).floatValue(); - } else { - return (char)left <= ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left <= (char)right; - } - } - - throw new ClassCastException("Cannot apply [<=] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static boolean gt(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() > ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() > ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() > ((Number)right).longValue(); - } else { - return ((Number)left).intValue() > ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() > (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() > (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() > (char)right; - } else { - return ((Number)left).intValue() > (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left > ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left > ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left > ((Number)right).floatValue(); - } else { - return (char)left > ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left > (char)right; - } - } - - throw new ClassCastException("Cannot apply [>] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } - - public static boolean gte(final Object left, final Object right) { - if (left instanceof Number) { - if (right instanceof Number) { - if (left instanceof Double || right instanceof Double) { - return ((Number)left).doubleValue() >= ((Number)right).doubleValue(); - } else if (left instanceof Float || right instanceof Float) { - return ((Number)left).floatValue() >= ((Number)right).floatValue(); - } else if (left instanceof Long || right instanceof Long) { - return ((Number)left).longValue() >= ((Number)right).longValue(); - } else { - return ((Number)left).intValue() >= ((Number)right).intValue(); - } - } else if (right instanceof Character) { - if (left instanceof Double) { - return ((Number)left).doubleValue() >= (char)right; - } else if (left instanceof Long) { - return ((Number)left).longValue() >= (char)right; - } else if (left instanceof Float) { - return ((Number)left).floatValue() >= (char)right; - } else { - return ((Number)left).intValue() >= (char)right; - } - } - } else if (left instanceof Character) { - if (right instanceof Number) { - if (right instanceof Double) { - return (char)left >= ((Number)right).doubleValue(); - } else if (right instanceof Long) { - return (char)left >= ((Number)right).longValue(); - } else if (right instanceof Float) { - return (char)left >= ((Number)right).floatValue(); - } else { - return (char)left >= ((Number)right).intValue(); - } - } else if (right instanceof Character) { - return (char)left >= (char)right; - } - } - - throw new ClassCastException("Cannot apply [>] operation to types " + - "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); - } // Conversion methods for Def to primitive types. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java index 5dc773672a7..c688b314243 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java @@ -31,9 +31,9 @@ import java.lang.invoke.MutableCallSite; /** * Painless invokedynamic bootstrap for the call site. *

- * Has 5 flavors (passed as static bootstrap parameters): dynamic method call, + * Has 7 flavors (passed as static bootstrap parameters): dynamic method call, * dynamic field load (getter), and dynamic field store (setter), dynamic array load, - * and dynamic array store. + * dynamic array store, iterator, and method reference. *

* When a new type is encountered at the call site, we lookup from the appropriate * whitelist, and cache with a guard. If we encounter too many types, we stop caching. @@ -60,6 +60,14 @@ public final class DefBootstrap { public static final int ARRAY_STORE = 4; /** static bootstrap parameter indicating a dynamic iteration, e.g. for (x : y) */ public static final int ITERATOR = 5; + /** static bootstrap parameter indicating a dynamic method reference, e.g. foo::bar */ + public static final int REFERENCE = 6; + /** static bootstrap parameter indicating a unary math operator, e.g. ~foo */ + public static final int UNARY_OPERATOR = 7; + /** static bootstrap parameter indicating a binary math operator, e.g. foo / bar */ + public static final int BINARY_OPERATOR = 8; + /** static bootstrap parameter indicating a shift operator, e.g. foo >> bar */ + public static final int SHIFT_OPERATOR = 9; /** * CallSite that implements the polymorphic inlining cache (PIC). @@ -71,17 +79,21 @@ public final class DefBootstrap { private final Lookup lookup; private final String name; private final int flavor; - private final long recipe; + private final Object[] args; int depth; // pkg-protected for testing - PIC(Lookup lookup, String name, MethodType type, int flavor, long recipe) { + PIC(Lookup lookup, String name, MethodType type, int flavor, Object[] args) { super(type); this.lookup = lookup; this.name = name; this.flavor = flavor; - this.recipe = recipe; - assert recipe == 0 || flavor == METHOD_CALL; - assert Long.bitCount(recipe) <= type.parameterCount(); + this.args = args; + + // For operators use a monomorphic cache, fallback is fast. + // Just start with a depth of MAX-1, to keep it a constant. + if (flavor == UNARY_OPERATOR || flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) { + depth = MAX_DEPTH - 1; + } final MethodHandle fallback = FALLBACK.bindTo(this) .asCollector(Object[].class, type.parameterCount()) @@ -97,27 +109,69 @@ public final class DefBootstrap { static boolean checkClass(Class clazz, Object receiver) { return receiver.getClass() == clazz; } + + /** + * guard method for inline caching: checks the receiver's class and the first argument + * are the same as the cached receiver and first argument. + */ + static boolean checkBinary(Class left, Class right, Object leftObject, Object rightObject) { + return leftObject.getClass() == left && rightObject.getClass() == right; + } + + /** + * guard method for inline caching: checks the first argument is the same + * as the cached first argument. + */ + static boolean checkBinaryArg(Class left, Class right, Object leftObject, Object rightObject) { + return rightObject.getClass() == right; + } /** * Does a slow lookup against the whitelist. */ - private MethodHandle lookup(int flavor, Class clazz, String name, Object[] args, long recipe) throws Throwable { + private MethodHandle lookup(int flavor, String name, Object[] args) throws Throwable { switch(flavor) { case METHOD_CALL: - return Def.lookupMethod(lookup, clazz, name, args, recipe); + return Def.lookupMethod(lookup, type(), args[0].getClass(), name, args, (Long) this.args[0]); case LOAD: - return Def.lookupGetter(clazz, name); + return Def.lookupGetter(args[0].getClass(), name); case STORE: - return Def.lookupSetter(clazz, name); + return Def.lookupSetter(args[0].getClass(), name); case ARRAY_LOAD: - return Def.lookupArrayLoad(clazz); + return Def.lookupArrayLoad(args[0].getClass()); case ARRAY_STORE: - return Def.lookupArrayStore(clazz); + return Def.lookupArrayStore(args[0].getClass()); case ITERATOR: - return Def.lookupIterator(clazz); + return Def.lookupIterator(args[0].getClass()); + case REFERENCE: + return Def.lookupReference(lookup, (String) this.args[0], args[0].getClass(), name); + case UNARY_OPERATOR: + case SHIFT_OPERATOR: + // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored) + return DefMath.lookupUnary(args[0].getClass(), name); + case BINARY_OPERATOR: + if (args[0] == null || args[1] == null) { + return getGeneric(flavor, name); // can handle nulls + } else { + return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name); + } default: throw new AssertionError(); } } + + /** + * Installs a permanent, generic solution that works with any parameter types, if possible. + */ + private MethodHandle getGeneric(int flavor, String name) throws Throwable { + switch(flavor) { + case UNARY_OPERATOR: + case BINARY_OPERATOR: + case SHIFT_OPERATOR: + return DefMath.lookupGeneric(name); + default: + return null; + } + } /** * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH} @@ -125,21 +179,56 @@ public final class DefBootstrap { */ @SuppressForbidden(reason = "slow path") Object fallback(Object[] args) throws Throwable { - final MethodType type = type(); - final Object receiver = args[0]; - final Class receiverClass = receiver.getClass(); - final MethodHandle target = lookup(flavor, receiverClass, name, args, recipe).asType(type); - if (depth >= MAX_DEPTH) { - // revert to a vtable call - setTarget(target); - return target.invokeWithArguments(args); + // caching defeated + MethodHandle generic = getGeneric(flavor, name); + if (generic != null) { + setTarget(generic.asType(type())); + return generic.invokeWithArguments(args); + } else { + return lookup(flavor, name, args).invokeWithArguments(args); + } + } + + final MethodType type = type(); + final MethodHandle target = lookup(flavor, name, args).asType(type); + + final MethodHandle test; + if (flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) { + // some binary operators support nulls, we handle them separate + Class clazz0 = args[0] == null ? null : args[0].getClass(); + Class clazz1 = args[1] == null ? null : args[1].getClass(); + if (type.parameterType(1) != Object.class) { + // case 1: only the receiver is unknown, just check that + MethodHandle unaryTest = CHECK_CLASS.bindTo(clazz0); + test = unaryTest.asType(unaryTest.type() + .changeParameterType(0, type.parameterType(0))); + } else if (type.parameterType(0) != Object.class) { + // case 2: only the argument is unknown, just check that + MethodHandle unaryTest = CHECK_BINARY_ARG.bindTo(clazz0).bindTo(clazz1); + test = unaryTest.asType(unaryTest.type() + .changeParameterType(0, type.parameterType(0)) + .changeParameterType(1, type.parameterType(1))); + } else { + // case 3: check both receiver and argument + MethodHandle binaryTest = CHECK_BINARY.bindTo(clazz0).bindTo(clazz1); + test = binaryTest.asType(binaryTest.type() + .changeParameterType(0, type.parameterType(0)) + .changeParameterType(1, type.parameterType(1))); + } + } else { + MethodHandle receiverTest = CHECK_CLASS.bindTo(args[0].getClass()); + test = receiverTest.asType(receiverTest.type() + .changeParameterType(0, type.parameterType(0))); } - MethodHandle test = CHECK_CLASS.bindTo(receiverClass); - test = test.asType(test.type().changeParameterType(0, type.parameterType(0))); - - final MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget()); + MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget()); + // very special cases, where even the receiver can be null (see JLS rules for string concat) + // we wrap + with an NPE catcher, and use our generic method in that case. + if (flavor == BINARY_OPERATOR && "add".equals(name) || "eq".equals(name)) { + MethodHandle handler = MethodHandles.dropArguments(getGeneric(flavor, name).asType(type()), 0, NullPointerException.class); + guard = MethodHandles.catchException(guard, NullPointerException.class, handler); + } depth++; @@ -148,12 +237,18 @@ public final class DefBootstrap { } private static final MethodHandle CHECK_CLASS; + private static final MethodHandle CHECK_BINARY; + private static final MethodHandle CHECK_BINARY_ARG; private static final MethodHandle FALLBACK; static { final Lookup lookup = MethodHandles.lookup(); try { CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass", - MethodType.methodType(boolean.class, Class.class, Object.class)); + MethodType.methodType(boolean.class, Class.class, Object.class)); + CHECK_BINARY = lookup.findStatic(lookup.lookupClass(), "checkBinary", + MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); + CHECK_BINARY_ARG = lookup.findStatic(lookup.lookupClass(), "checkBinaryArg", + MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback", MethodType.methodType(Object.class, Object[].class)); } catch (ReflectiveOperationException e) { @@ -170,8 +265,36 @@ public final class DefBootstrap { *

* see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic */ - public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, long recipe) { - return new PIC(lookup, name, type, flavor, recipe); + public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, Object... args) { + // validate arguments + switch(flavor) { + case METHOD_CALL: + if (args.length != 1) { + throw new BootstrapMethodError("Invalid number of parameters for method call"); + } + if (args[0] instanceof Long == false) { + throw new BootstrapMethodError("Illegal parameter for method call: " + args[0]); + } + long recipe = (Long) args[0]; + if (Long.bitCount(recipe) > type.parameterCount()) { + throw new BootstrapMethodError("Illegal recipe for method call: too many bits"); + } + break; + case REFERENCE: + if (args.length != 1) { + throw new BootstrapMethodError("Invalid number of parameters for reference call"); + } + if (args[0] instanceof String == false) { + throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]); + } + break; + default: + if (args.length > 0) { + throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor); + } + break; + } + return new PIC(lookup, name, type, flavor, args); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java new file mode 100644 index 00000000000..e6b3a8c6003 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java @@ -0,0 +1,1149 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.MethodHandles.Lookup; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Dynamic operators for painless. + *

+ * Each operator must "support" the following types: + * {@code int,long,float,double,boolean,Object}. Operators can throw exceptions if + * the type is illegal. The {@code Object} type must be a "generic" handler that + * handles all legal types: it must be convertible to every possible legal signature. + */ +@SuppressWarnings("unused") +public class DefMath { + + // Unary not: only applicable to integral types + + private static int not(int v) { + return ~v; + } + + private static long not(long v) { + return ~v; + } + + private static float not(float v) { + throw new ClassCastException("Cannot apply not [~] to type [float]"); + } + + private static double not(double v) { + throw new ClassCastException("Cannot apply not [~] to type [double]"); + } + + private static boolean not(boolean v) { + throw new ClassCastException("Cannot apply not [~] to type [boolean]"); + } + + private static Object not(Object unary) { + if (unary instanceof Long) { + return ~(Long)unary; + } else if (unary instanceof Integer) { + return ~(Integer)unary; + } else if (unary instanceof Short) { + return ~(Short)unary; + } else if (unary instanceof Character) { + return ~(Character)unary; + } else if (unary instanceof Byte) { + return ~(Byte)unary; + } + + throw new ClassCastException("Cannot apply [~] operation to type " + + "[" + unary.getClass().getCanonicalName() + "]."); + } + + // unary negation and plus: applicable to all numeric types + + private static int neg(int v) { + return -v; + } + + private static long neg(long v) { + return -v; + } + + private static float neg(float v) { + return -v; + } + + private static double neg(double v) { + return -v; + } + + private static boolean neg(boolean v) { + throw new ClassCastException("Cannot apply [-] operation to type [boolean]"); + } + + private static Object neg(final Object unary) { + if (unary instanceof Double) { + return -(double)unary; + } else if (unary instanceof Long) { + return -(long)unary; + } else if (unary instanceof Integer) { + return -(int)unary; + } else if (unary instanceof Float) { + return -(float)unary; + } else if (unary instanceof Short) { + return -(short)unary; + } else if (unary instanceof Character) { + return -(char)unary; + } else if (unary instanceof Byte) { + return -(byte)unary; + } + + throw new ClassCastException("Cannot apply [-] operation to type " + + "[" + unary.getClass().getCanonicalName() + "]."); + } + + private static int plus(int v) { + return +v; + } + + private static long plus(long v) { + return +v; + } + + private static float plus(float v) { + return +v; + } + + private static double plus(double v) { + return +v; + } + + private static boolean plus(boolean v) { + throw new ClassCastException("Cannot apply [+] operation to type [boolean]"); + } + + private static Object plus(final Object unary) { + if (unary instanceof Double) { + return +(double)unary; + } else if (unary instanceof Long) { + return +(long)unary; + } else if (unary instanceof Integer) { + return +(int)unary; + } else if (unary instanceof Float) { + return +(float)unary; + } else if (unary instanceof Short) { + return +(short)unary; + } else if (unary instanceof Character) { + return +(char)unary; + } else if (unary instanceof Byte) { + return +(byte)unary; + } + + throw new ClassCastException("Cannot apply [+] operation to type " + + "[" + unary.getClass().getCanonicalName() + "]."); + } + + // multiplication/division/remainder/subtraction: applicable to all integer types + + private static int mul(int a, int b) { + return a * b; + } + + private static long mul(long a, long b) { + return a * b; + } + + private static float mul(float a, float b) { + return a * b; + } + + private static double mul(double a, double b) { + return a * b; + } + + private static boolean mul(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [*] operation to type [boolean]"); + } + + private static Object mul(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() * ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() * ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() * ((Number)right).longValue(); + } else { + return ((Number)left).intValue() * ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() * (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() * (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() * (char)right; + } else { + return ((Number)left).intValue() * (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left * ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left * ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left * ((Number)right).floatValue(); + } else { + return (char)left * ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left * (char)right; + } + } + + throw new ClassCastException("Cannot apply [*] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static int div(int a, int b) { + return a / b; + } + + private static long div(long a, long b) { + return a / b; + } + + private static float div(float a, float b) { + return a / b; + } + + private static double div(double a, double b) { + return a / b; + } + + private static boolean div(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [/] operation to type [boolean]"); + } + + private static Object div(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() / ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() / ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() / ((Number)right).longValue(); + } else { + return ((Number)left).intValue() / ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() / (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() / (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() / (char)right; + } else { + return ((Number)left).intValue() / (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left / ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left / ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left / ((Number)right).floatValue(); + } else { + return (char)left / ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left / (char)right; + } + } + + throw new ClassCastException("Cannot apply [/] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static int rem(int a, int b) { + return a % b; + } + + private static long rem(long a, long b) { + return a % b; + } + + private static float rem(float a, float b) { + return a % b; + } + + private static double rem(double a, double b) { + return a % b; + } + + private static boolean rem(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [%] operation to type [boolean]"); + } + + private static Object rem(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() % ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() % ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() % ((Number)right).longValue(); + } else { + return ((Number)left).intValue() % ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() % (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() % (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() % (char)right; + } else { + return ((Number)left).intValue() % (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left % ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left % ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left % ((Number)right).floatValue(); + } else { + return (char)left % ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left % (char)right; + } + } + + throw new ClassCastException("Cannot apply [%] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + // addition: applicable to all numeric types. + // additionally, if either type is a string, the other type can be any arbitrary type (including null) + + private static int add(int a, int b) { + return a + b; + } + + private static long add(long a, long b) { + return a + b; + } + + private static float add(float a, float b) { + return a + b; + } + + private static double add(double a, double b) { + return a + b; + } + + private static boolean add(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [+] operation to type [boolean]"); + } + + private static Object add(Object left, Object right) { + if (left instanceof String || right instanceof String) { + return "" + left + right; + } else if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() + ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() + ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() + ((Number)right).longValue(); + } else { + return ((Number)left).intValue() + ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() + (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() + (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() + (char)right; + } else { + return ((Number)left).intValue() + (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left + ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left + ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left + ((Number)right).floatValue(); + } else { + return (char)left + ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left + (char)right; + } + } + + throw new ClassCastException("Cannot apply [+] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static int sub(int a, int b) { + return a - b; + } + + private static long sub(long a, long b) { + return a - b; + } + + private static float sub(float a, float b) { + return a - b; + } + + private static double sub(double a, double b) { + return a - b; + } + + private static boolean sub(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [-] operation to type [boolean]"); + } + + private static Object sub(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() - ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() - ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() - ((Number)right).longValue(); + } else { + return ((Number)left).intValue() - ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() - (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() - (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() - (char)right; + } else { + return ((Number)left).intValue() - (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left - ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left - ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left - ((Number)right).floatValue(); + } else { + return (char)left - ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left - (char)right; + } + } + + throw new ClassCastException("Cannot apply [-] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + // eq: applicable to any arbitrary type, including nulls for both arguments!!! + + private static boolean eq(int a, int b) { + return a == b; + } + + private static boolean eq(long a, long b) { + return a == b; + } + + private static boolean eq(float a, float b) { + return a == b; + } + + private static boolean eq(double a, double b) { + return a == b; + } + + private static boolean eq(boolean a, boolean b) { + return a == b; + } + + private static boolean eq(Object left, Object right) { + if (left != null && right != null) { + if (left instanceof Double) { + if (right instanceof Number) { + return (double)left == ((Number)right).doubleValue(); + } else if (right instanceof Character) { + return (double)left == (char)right; + } + } else if (right instanceof Double) { + if (left instanceof Number) { + return ((Number)left).doubleValue() == (double)right; + } else if (left instanceof Character) { + return (char)left == ((Number)right).doubleValue(); + } + } else if (left instanceof Float) { + if (right instanceof Number) { + return (float)left == ((Number)right).floatValue(); + } else if (right instanceof Character) { + return (float)left == (char)right; + } + } else if (right instanceof Float) { + if (left instanceof Number) { + return ((Number)left).floatValue() == (float)right; + } else if (left instanceof Character) { + return (char)left == ((Number)right).floatValue(); + } + } else if (left instanceof Long) { + if (right instanceof Number) { + return (long)left == ((Number)right).longValue(); + } else if (right instanceof Character) { + return (long)left == (char)right; + } + } else if (right instanceof Long) { + if (left instanceof Number) { + return ((Number)left).longValue() == (long)right; + } else if (left instanceof Character) { + return (char)left == ((Number)right).longValue(); + } + } else if (left instanceof Number) { + if (right instanceof Number) { + return ((Number)left).intValue() == ((Number)right).intValue(); + } else if (right instanceof Character) { + return ((Number)left).intValue() == (char)right; + } + } else if (right instanceof Number && left instanceof Character) { + return (char)left == ((Number)right).intValue(); + } else if (left instanceof Character && right instanceof Character) { + return (char)left == (char)right; + } + + return left.equals(right); + } + + return left == null && right == null; + } + + // comparison operators: applicable for any numeric type + + private static boolean lt(int a, int b) { + return a < b; + } + + private static boolean lt(long a, long b) { + return a < b; + } + + private static boolean lt(float a, float b) { + return a < b; + } + + private static boolean lt(double a, double b) { + return a < b; + } + + private static boolean lt(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [<] operation to type [boolean]"); + } + + private static boolean lt(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() < ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() < ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() < ((Number)right).longValue(); + } else { + return ((Number)left).intValue() < ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() < (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() < (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() < (char)right; + } else { + return ((Number)left).intValue() < (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left < ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left < ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left < ((Number)right).floatValue(); + } else { + return (char)left < ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left < (char)right; + } + } + + throw new ClassCastException("Cannot apply [<] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static boolean lte(int a, int b) { + return a <= b; + } + + private static boolean lte(long a, long b) { + return a <= b; + } + + private static boolean lte(float a, float b) { + return a <= b; + } + + private static boolean lte(double a, double b) { + return a <= b; + } + + private static boolean lte(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [<=] operation to type [boolean]"); + } + + private static boolean lte(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() <= ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() <= ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() <= ((Number)right).longValue(); + } else { + return ((Number)left).intValue() <= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() <= (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() <= (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() <= (char)right; + } else { + return ((Number)left).intValue() <= (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left <= ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left <= ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left <= ((Number)right).floatValue(); + } else { + return (char)left <= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left <= (char)right; + } + } + + throw new ClassCastException("Cannot apply [<=] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static boolean gt(int a, int b) { + return a > b; + } + + private static boolean gt(long a, long b) { + return a > b; + } + + private static boolean gt(float a, float b) { + return a > b; + } + + private static boolean gt(double a, double b) { + return a > b; + } + + private static boolean gt(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [>] operation to type [boolean]"); + } + + private static boolean gt(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() > ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() > ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() > ((Number)right).longValue(); + } else { + return ((Number)left).intValue() > ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() > (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() > (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() > (char)right; + } else { + return ((Number)left).intValue() > (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left > ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left > ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left > ((Number)right).floatValue(); + } else { + return (char)left > ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left > (char)right; + } + } + + throw new ClassCastException("Cannot apply [>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + private static boolean gte(int a, int b) { + return a >= b; + } + + private static boolean gte(long a, long b) { + return a >= b; + } + + private static boolean gte(float a, float b) { + return a >= b; + } + + private static boolean gte(double a, double b) { + return a >= b; + } + + private static boolean gte(boolean a, boolean b) { + throw new ClassCastException("Cannot apply [>=] operation to type [boolean]"); + } + + private static boolean gte(Object left, Object right) { + if (left instanceof Number) { + if (right instanceof Number) { + if (left instanceof Double || right instanceof Double) { + return ((Number)left).doubleValue() >= ((Number)right).doubleValue(); + } else if (left instanceof Float || right instanceof Float) { + return ((Number)left).floatValue() >= ((Number)right).floatValue(); + } else if (left instanceof Long || right instanceof Long) { + return ((Number)left).longValue() >= ((Number)right).longValue(); + } else { + return ((Number)left).intValue() >= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + if (left instanceof Double) { + return ((Number)left).doubleValue() >= (char)right; + } else if (left instanceof Long) { + return ((Number)left).longValue() >= (char)right; + } else if (left instanceof Float) { + return ((Number)left).floatValue() >= (char)right; + } else { + return ((Number)left).intValue() >= (char)right; + } + } + } else if (left instanceof Character) { + if (right instanceof Number) { + if (right instanceof Double) { + return (char)left >= ((Number)right).doubleValue(); + } else if (right instanceof Long) { + return (char)left >= ((Number)right).longValue(); + } else if (right instanceof Float) { + return (char)left >= ((Number)right).floatValue(); + } else { + return (char)left >= ((Number)right).intValue(); + } + } else if (right instanceof Character) { + return (char)left >= (char)right; + } + } + + throw new ClassCastException("Cannot apply [>] operation to types " + + "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "]."); + } + + // helper methods to convert an integral according to numeric promotion + // this is used by the generic code for bitwise and shift operators + + private static long longIntegralValue(Object o) { + if (o instanceof Long) { + return (long)o; + } else if (o instanceof Integer || o instanceof Short || o instanceof Byte) { + return ((Number)o).longValue(); + } else if (o instanceof Character) { + return (char)o; + } else { + throw new ClassCastException("Cannot convert [" + o.getClass().getCanonicalName() + "] to an integral value."); + } + } + + private static int intIntegralValue(Object o) { + if (o instanceof Integer || o instanceof Short || o instanceof Byte) { + return ((Number)o).intValue(); + } else if (o instanceof Character) { + return (char)o; + } else { + throw new ClassCastException("Cannot convert [" + o.getClass().getCanonicalName() + "] to an integral value."); + } + } + + // bitwise operators: valid only for integral types + + private static int and(int a, int b) { + return a & b; + } + + private static long and(long a, long b) { + return a & b; + } + + private static float and(float a, float b) { + throw new ClassCastException("Cannot apply [&] operation to type [float]"); + } + + private static double and(double a, double b) { + throw new ClassCastException("Cannot apply [&] operation to type [float]"); + } + + private static boolean and(boolean a, boolean b) { + return a & b; + } + + private static Object and(Object left, Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left & (boolean)right; + } else if (left instanceof Long || right instanceof Long) { + return longIntegralValue(left) & longIntegralValue(right); + } else { + return intIntegralValue(left) & intIntegralValue(right); + } + } + + private static int xor(int a, int b) { + return a ^ b; + } + + private static long xor(long a, long b) { + return a ^ b; + } + + private static float xor(float a, float b) { + throw new ClassCastException("Cannot apply [^] operation to type [float]"); + } + + private static double xor(double a, double b) { + throw new ClassCastException("Cannot apply [^] operation to type [float]"); + } + + private static boolean xor(boolean a, boolean b) { + return a ^ b; + } + + private static Object xor(Object left, Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left ^ (boolean)right; + } else if (left instanceof Long || right instanceof Long) { + return longIntegralValue(left) ^ longIntegralValue(right); + } else { + return intIntegralValue(left) ^ intIntegralValue(right); + } + } + + private static int or(int a, int b) { + return a | b; + } + + private static long or(long a, long b) { + return a | b; + } + + private static float or(float a, float b) { + throw new ClassCastException("Cannot apply [|] operation to type [float]"); + } + + private static double or(double a, double b) { + throw new ClassCastException("Cannot apply [|] operation to type [float]"); + } + + private static boolean or(boolean a, boolean b) { + return a | b; + } + + private static Object or(Object left, Object right) { + if (left instanceof Boolean && right instanceof Boolean) { + return (boolean)left | (boolean)right; + } else if (left instanceof Long || right instanceof Long) { + return longIntegralValue(left) | longIntegralValue(right); + } else { + return intIntegralValue(left) | intIntegralValue(right); + } + } + + // shift operators, valid for any integral types, but does not promote. + // we implement all shifts as long shifts, because the extra bits are ignored anyway. + + private static int lsh(int a, long b) { + return a << b; + } + + private static long lsh(long a, long b) { + return a << b; + } + + private static float lsh(float a, long b) { + throw new ClassCastException("Cannot apply [<<] operation to type [float]"); + } + + private static double lsh(double a, long b) { + throw new ClassCastException("Cannot apply [<<] operation to type [double]"); + } + + private static boolean lsh(boolean a, long b) { + throw new ClassCastException("Cannot apply [<<] operation to type [boolean]"); + } + + public static Object lsh(Object left, long right) { + if (left instanceof Long) { + return (long)(left) << right; + } else { + return intIntegralValue(left) << right; + } + } + + private static int rsh(int a, long b) { + return a >> b; + } + + private static long rsh(long a, long b) { + return a >> b; + } + + private static float rsh(float a, long b) { + throw new ClassCastException("Cannot apply [>>] operation to type [float]"); + } + + private static double rsh(double a, long b) { + throw new ClassCastException("Cannot apply [>>] operation to type [double]"); + } + + private static boolean rsh(boolean a, long b) { + throw new ClassCastException("Cannot apply [>>] operation to type [boolean]"); + } + + public static Object rsh(Object left, long right) { + if (left instanceof Long) { + return (long)left >> right; + } else { + return intIntegralValue(left) >> right; + } + } + + private static int ush(int a, long b) { + return a >>> b; + } + + private static long ush(long a, long b) { + return a >>> b; + } + + private static float ush(float a, long b) { + throw new ClassCastException("Cannot apply [>>>] operation to type [float]"); + } + + private static double ush(double a, long b) { + throw new ClassCastException("Cannot apply [>>>] operation to type [double]"); + } + + private static boolean ush(boolean a, long b) { + throw new ClassCastException("Cannot apply [>>>] operation to type [boolean]"); + } + + public static Object ush(Object left, long right) { + if (left instanceof Long) { + return (long)(left) >>> right; + } else { + return intIntegralValue(left) >>> right; + } + } + + /** + * unboxes a class to its primitive type, or returns the original + * class if its not a boxed type. + */ + private static Class unbox(Class clazz) { + if (clazz == Boolean.class) { + return boolean.class; + } else if (clazz == Byte.class) { + return byte.class; + } else if (clazz == Short.class) { + return short.class; + } else if (clazz == Character.class) { + return char.class; + } else if (clazz == Integer.class) { + return int.class; + } else if (clazz == Long.class) { + return long.class; + } else if (clazz == Float.class) { + return float.class; + } else if (clazz == Double.class) { + return double.class; + } else { + return clazz; + } + } + + /** Unary promotion. All Objects are promoted to Object. */ + private static Class promote(Class clazz) { + // if either is a non-primitive type -> Object. + if (clazz.isPrimitive() == false) { + return Object.class; + } + // always promoted to integer + if (clazz == byte.class || clazz == short.class || clazz == char.class || clazz == int.class) { + return int.class; + } else { + return clazz; + } + } + + /** Binary promotion. */ + private static Class promote(Class a, Class b) { + // if either is a non-primitive type -> Object. + if (a.isPrimitive() == false || b.isPrimitive() == false) { + return Object.class; + } + + // boolean -> boolean + if (a == boolean.class && b == boolean.class) { + return boolean.class; + } + + // ordinary numeric promotion + if (a == double.class || b == double.class) { + return double.class; + } else if (a == float.class || b == float.class) { + return float.class; + } else if (a == long.class || b == long.class) { + return long.class; + } else { + return int.class; + } + } + + private static final Lookup PRIV_LOOKUP = MethodHandles.lookup(); + + private static final Map,Map> TYPE_OP_MAPPING = Collections.unmodifiableMap( + Stream.of(boolean.class, int.class, long.class, float.class, double.class, Object.class) + .collect(Collectors.toMap(Function.identity(), type -> { + try { + Map map = new HashMap<>(); + MethodType unary = MethodType.methodType(type, type); + MethodType binary = MethodType.methodType(type, type, type); + MethodType comparison = MethodType.methodType(boolean.class, type, type); + MethodType shift = MethodType.methodType(type, type, long.class); + Class clazz = PRIV_LOOKUP.lookupClass(); + map.put("not", PRIV_LOOKUP.findStatic(clazz, "not", unary)); + map.put("neg", PRIV_LOOKUP.findStatic(clazz, "neg", unary)); + map.put("plus", PRIV_LOOKUP.findStatic(clazz, "plus", unary)); + map.put("mul", PRIV_LOOKUP.findStatic(clazz, "mul", binary)); + map.put("div", PRIV_LOOKUP.findStatic(clazz, "div", binary)); + map.put("rem", PRIV_LOOKUP.findStatic(clazz, "rem", binary)); + map.put("add", PRIV_LOOKUP.findStatic(clazz, "add", binary)); + map.put("sub", PRIV_LOOKUP.findStatic(clazz, "sub", binary)); + map.put("and", PRIV_LOOKUP.findStatic(clazz, "and", binary)); + map.put("or", PRIV_LOOKUP.findStatic(clazz, "or", binary)); + map.put("xor", PRIV_LOOKUP.findStatic(clazz, "xor", binary)); + map.put("eq", PRIV_LOOKUP.findStatic(clazz, "eq", comparison)); + map.put("lt", PRIV_LOOKUP.findStatic(clazz, "lt", comparison)); + map.put("lte", PRIV_LOOKUP.findStatic(clazz, "lte", comparison)); + map.put("gt", PRIV_LOOKUP.findStatic(clazz, "gt", comparison)); + map.put("gte", PRIV_LOOKUP.findStatic(clazz, "gte", comparison)); + map.put("lsh", PRIV_LOOKUP.findStatic(clazz, "lsh", shift)); + map.put("rsh", PRIV_LOOKUP.findStatic(clazz, "rsh", shift)); + map.put("ush", PRIV_LOOKUP.findStatic(clazz, "ush", shift)); + return map; + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + })) + ); + + /** Returns an appropriate method handle for a unary or shift operator, based only on the receiver (LHS) */ + public static MethodHandle lookupUnary(Class receiverClass, String name) { + MethodHandle handle = TYPE_OP_MAPPING.get(promote(unbox(receiverClass))).get(name); + if (handle == null) { + throw new ClassCastException("Cannot apply operator [" + name + "] to type [" + receiverClass + "]"); + } + return handle; + } + + /** Returns an appropriate method handle for a binary operator, based only promotion of the LHS and RHS arguments */ + public static MethodHandle lookupBinary(Class classA, Class classB, String name) { + MethodHandle handle = TYPE_OP_MAPPING.get(promote(promote(unbox(classA)), promote(unbox(classB)))).get(name); + if (handle == null) { + throw new ClassCastException("Cannot apply operator [" + name + "] to types [" + classA + "] and [" + classB + "]"); + } + return handle; + } + + /** Returns a generic method handle for any operator, that can handle all valid signatures, nulls, corner cases */ + public static MethodHandle lookupGeneric(String name) { + return TYPE_OP_MAPPING.get(Object.class).get(name); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java index dbfc9993c31..7188791d971 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java @@ -27,6 +27,7 @@ import java.io.InputStreamReader; import java.io.LineNumberReader; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -45,9 +46,9 @@ import java.util.Spliterator; * methods and fields during at both compile-time and runtime. */ public final class Definition { - + private static final List DEFINITION_FILES = Collections.unmodifiableList( - Arrays.asList("org.elasticsearch.txt", + Arrays.asList("org.elasticsearch.txt", "java.lang.txt", "java.math.txt", "java.text.txt", @@ -58,6 +59,7 @@ public final class Definition { "java.time.zone.txt", "java.util.txt", "java.util.function.txt", + "java.util.regex.txt", "java.util.stream.txt", "joda.time.txt")); @@ -188,8 +190,8 @@ public final class Definition { public final int modifiers; public final MethodHandle handle; - private Method(String name, Struct owner, Type rtn, List arguments, - org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) { + public Method(String name, Struct owner, Type rtn, List arguments, + org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) { this.name = name; this.owner = owner; this.rtn = rtn; @@ -198,6 +200,46 @@ public final class Definition { this.modifiers = modifiers; this.handle = handle; } + + /** + * Returns MethodType for this method. + *

+ * This works even for user-defined Methods (where the MethodHandle is null). + */ + public MethodType getMethodType() { + // we have a methodhandle already (e.g. whitelisted class) + // just return its type + if (handle != null) { + return handle.type(); + } + // otherwise compute it + final Class params[]; + final Class returnValue; + if (Modifier.isStatic(modifiers)) { + // static method: straightforward copy + params = new Class[arguments.size()]; + for (int i = 0; i < arguments.size(); i++) { + params[i] = arguments.get(i).clazz; + } + returnValue = rtn.clazz; + } else if ("".equals(name)) { + // constructor: returns the owner class + params = new Class[arguments.size()]; + for (int i = 0; i < arguments.size(); i++) { + params[i] = arguments.get(i).clazz; + } + returnValue = owner.clazz; + } else { + // virtual/interface method: add receiver class + params = new Class[1 + arguments.size()]; + params[0] = owner.clazz; + for (int i = 0; i < arguments.size(); i++) { + params[i + 1] = arguments.get(i).clazz; + } + returnValue = rtn.clazz; + } + return MethodType.methodType(returnValue, params); + } } public static final class Field { @@ -286,7 +328,7 @@ public final class Definition { public final Map staticMembers; public final Map members; - + private final SetOnce functionalMethod; private Struct(final String name, final Class clazz, final org.objectweb.asm.Type type) { @@ -300,8 +342,8 @@ public final class Definition { staticMembers = new HashMap<>(); members = new HashMap<>(); - - functionalMethod = new SetOnce(); + + functionalMethod = new SetOnce<>(); } private Struct(final Struct struct) { @@ -315,7 +357,7 @@ public final class Definition { staticMembers = Collections.unmodifiableMap(struct.staticMembers); members = Collections.unmodifiableMap(struct.members); - + functionalMethod = struct.functionalMethod; } @@ -342,8 +384,8 @@ public final class Definition { public int hashCode() { return name.hashCode(); } - - /** + + /** * If this class is a functional interface according to JLS, returns its method. * Otherwise returns null. */ @@ -637,7 +679,7 @@ public final class Definition { final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect); final Type returnType = getTypeInternal("void"); final MethodHandle handle; - + try { handle = MethodHandles.publicLookup().in(owner.clazz).unreflectConstructor(reflect); } catch (final IllegalAccessException exception) { @@ -645,7 +687,7 @@ public final class Definition { " not found for class [" + owner.clazz.getName() + "]" + " with arguments " + Arrays.toString(classes) + "."); } - + final Method constructor = new Method(name, owner, returnType, Arrays.asList(args), asm, reflect.getModifiers(), handle); owner.constructors.put(methodKey, constructor); @@ -755,7 +797,7 @@ public final class Definition { " method [" + name + "]" + " within the struct [" + owner.name + "]."); } - + final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect); MethodHandle handle; @@ -856,7 +898,7 @@ public final class Definition { throw new ClassCastException("Child struct [" + child.name + "]" + " is not a super type of owner struct [" + owner.name + "] in copy."); } - + for (Map.Entry kvPair : child.methods.entrySet()) { MethodKey methodKey = kvPair.getKey(); Method method = kvPair.getValue(); @@ -890,8 +932,7 @@ public final class Definition { throw new AssertionError(e); } } - owner.methods.put(methodKey, - new Method(method.name, owner, method.rtn, method.arguments, method.method, method.modifiers, method.handle)); + owner.methods.put(methodKey, method); } } @@ -954,7 +995,7 @@ public final class Definition { runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters)); } - + /** computes the functional interface method for a class, or returns null */ private Method computeFunctionalInterfaceMethod(Struct clazz) { if (!clazz.clazz.isInterface()) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java index bef83daaad3..603023e61fe 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java @@ -1,5 +1,7 @@ package org.elasticsearch.painless; +import java.util.function.Function; + /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -63,4 +65,9 @@ public class FeatureTest { public static boolean overloadedStatic(boolean whatToReturn) { return whatToReturn; } + + /** method taking two functions! */ + public Object twoFunctionsOfX(Function f, Function g) { + return f.apply(g.apply(x)); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index 03b6cc604c9..9015b4dee61 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -49,13 +49,83 @@ public class FunctionRef { public final Handle implMethodASM; /** - * Creates a new FunctionRef. + * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist. * @param expected interface type to implement. * @param type the left hand side of a method reference expression * @param call the right hand side of a method reference expression + * @param captures captured arguments + */ + public FunctionRef(Definition.Type expected, String type, String call, Class... captures) { + this(expected, expected.struct.getFunctionalMethod(), lookup(expected, type, call, captures.length > 0), captures); + } + + /** + * Creates a new FunctionRef (already resolved) + * @param expected interface type to implement + * @param method functional interface method + * @param impl implementation method + * @param captures captured arguments + */ + public FunctionRef(Definition.Type expected, Definition.Method method, Definition.Method impl, Class... captures) { + // e.g. compareTo + invokedName = method.name; + // e.g. (Object)Comparator + invokedType = MethodType.methodType(expected.clazz, captures); + // e.g. (Object,Object)int + interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1); + + final int tag; + if ("".equals(impl.name)) { + tag = Opcodes.H_NEWINVOKESPECIAL; + } else if (Modifier.isStatic(impl.modifiers)) { + tag = Opcodes.H_INVOKESTATIC; + } else if (impl.owner.clazz.isInterface()) { + tag = Opcodes.H_INVOKEINTERFACE; + } else { + tag = Opcodes.H_INVOKEVIRTUAL; + } + final String owner; + final boolean ownerIsInterface; + if (impl.owner == null) { + // owner == null: script class itself + ownerIsInterface = false; + owner = WriterConstants.CLASS_TYPE.getInternalName(); + } else { + ownerIsInterface = impl.owner.clazz.isInterface(); + owner = impl.owner.type.getInternalName(); + } + implMethodASM = new Handle(tag, owner, impl.name, impl.method.getDescriptor(), ownerIsInterface); + implMethod = impl.handle; + + // remove any prepended captured arguments for the 'natural' signature. + samMethodType = impl.getMethodType().dropParameterTypes(0, captures.length); + } + + /** + * Creates a new FunctionRef (low level). + *

+ * This will not set implMethodASM. It is for runtime use only. */ - public FunctionRef(Definition.Type expected, String type, String call) { - boolean isCtorReference = "new".equals(call); + public FunctionRef(Definition.Type expected, Definition.Method method, MethodHandle impl, Class... captures) { + // e.g. compareTo + invokedName = method.name; + // e.g. (Object)Comparator + invokedType = MethodType.methodType(expected.clazz, captures); + // e.g. (Object,Object)int + interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1); + + implMethod = impl; + + implMethodASM = null; + + // remove any prepended captured arguments for the 'natural' signature. + samMethodType = impl.type().dropParameterTypes(0, captures.length); + } + + /** + * Looks up {@code type::call} from the whitelist, and returns a matching method. + */ + private static Definition.Method lookup(Definition.Type expected, String type, String call, boolean receiverCaptured) { // check its really a functional interface // for e.g. Comparable Method method = expected.struct.getFunctionalMethod(); @@ -63,24 +133,27 @@ public class FunctionRef { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + "to [" + expected.name + "], not a functional interface"); } - // e.g. compareTo - invokedName = method.name; - // e.g. (Object)Comparator - invokedType = MethodType.methodType(expected.clazz); - // e.g. (Object,Object)int - interfaceMethodType = method.handle.type().dropParameterTypes(0, 1); + // lookup requested method Definition.Struct struct = Definition.getType(type).struct; final Definition.Method impl; // ctor ref - if (isCtorReference) { + if ("new".equals(call)) { impl = struct.constructors.get(new Definition.MethodKey("", method.arguments.size())); } else { // look for a static impl first Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.arguments.size())); if (staticImpl == null) { // otherwise a virtual impl - impl = struct.methods.get(new Definition.MethodKey(call, method.arguments.size()-1)); + final int arity; + if (receiverCaptured) { + // receiver captured + arity = method.arguments.size(); + } else { + // receiver passed + arity = method.arguments.size() - 1; + } + impl = struct.methods.get(new Definition.MethodKey(call, arity)); } else { impl = staticImpl; } @@ -89,27 +162,9 @@ public class FunctionRef { throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " + "[" + expected + "]"); } - - final int tag; - if (isCtorReference) { - tag = Opcodes.H_NEWINVOKESPECIAL; - } else if (Modifier.isStatic(impl.modifiers)) { - tag = Opcodes.H_INVOKESTATIC; - } else { - tag = Opcodes.H_INVOKEVIRTUAL; - } - implMethodASM = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor()); - implMethod = impl.handle; - if (isCtorReference) { - samMethodType = MethodType.methodType(interfaceMethodType.returnType(), impl.handle.type().parameterArray()); - } else if (Modifier.isStatic(impl.modifiers)) { - samMethodType = impl.handle.type(); - } else { - // ensure the receiver type is exact and not a superclass type - samMethodType = impl.handle.type().changeParameterType(0, struct.clazz); - } + return impl; } - + /** Returns true if you should ask LambdaMetaFactory to construct a bridge for the interface signature */ public boolean needsBridges() { // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking? diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java new file mode 100644 index 00000000000..30f15c42dde --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java @@ -0,0 +1,364 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.Definition.Type; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Tracks user defined methods and variables across compilation phases. + */ +public final class Locals { + + /** + * Tracks reserved variables. Must be given to any source of input + * prior to beginning the analysis phase so that reserved variables + * are known ahead of time to assign appropriate slots without + * being wasteful. + */ + public interface Reserved { + void markReserved(String name); + boolean isReserved(String name); + + void setMaxLoopCounter(int max); + int getMaxLoopCounter(); + } + + public static final class ExecuteReserved implements Reserved { + public static final String THIS = "#this"; + public static final String PARAMS = "params"; + public static final String SCORER = "#scorer"; + public static final String DOC = "doc"; + public static final String VALUE = "_value"; + public static final String SCORE = "_score"; + public static final String CTX = "ctx"; + public static final String LOOP = "#loop"; + + private boolean score = false; + private boolean ctx = false; + private int maxLoopCounter = 0; + + @Override + public void markReserved(String name) { + if (SCORE.equals(name)) { + score = true; + } else if (CTX.equals(name)) { + ctx = true; + } + } + + @Override + public boolean isReserved(String name) { + return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) || + name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP); + } + + public boolean usesScore() { + return score; + } + + public boolean usesCtx() { + return ctx; + } + + @Override + public void setMaxLoopCounter(int max) { + maxLoopCounter = max; + } + + @Override + public int getMaxLoopCounter() { + return maxLoopCounter; + } + } + + public static final class FunctionReserved implements Reserved { + public static final String THIS = "#this"; + public static final String LOOP = "#loop"; + + private int maxLoopCounter = 0; + + public void markReserved(String name) { + // Do nothing. + } + + public boolean isReserved(String name) { + return name.equals(THIS) || name.equals(LOOP); + } + + @Override + public void setMaxLoopCounter(int max) { + maxLoopCounter = max; + } + + @Override + public int getMaxLoopCounter() { + return maxLoopCounter; + } + } + + public static final class Variable { + public final Location location; + public final String name; + public final Type type; + public final int slot; + public final boolean readonly; + + public boolean read = false; + + private Variable(Location location, String name, Type type, int slot, boolean readonly) { + this.location = location; + this.name = name; + this.type = type; + this.slot = slot; + this.readonly = readonly; + } + } + + public static final class Constant { + public final Location location; + public final String name; + public final org.objectweb.asm.Type type; + public final Consumer initializer; + + private Constant(Location location, String name, org.objectweb.asm.Type type, Consumer initializer) { + this.location = location; + this.name = name; + this.type = type; + this.initializer = initializer; + } + } + + public static final class Parameter { + public final Location location; + public final String name; + public final Type type; + + public Parameter(Location location, String name, Type type) { + this.location = location; + this.name = name; + this.type = type; + } + } + + private final Reserved reserved; + private final Map methods; + private final Map constants; + private final Type rtnType; + + // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead? + private final Deque scopes = new ArrayDeque<>(); + private final Deque variables = new ArrayDeque<>(); + + public Locals(ExecuteReserved reserved, Map methods) { + this.reserved = reserved; + this.methods = Collections.unmodifiableMap(methods); + this.constants = new HashMap<>(); + this.rtnType = Definition.OBJECT_TYPE; + + incrementScope(); + + // Method variables. + + // This reference. Internal use only. + addVariable(null, Definition.getType("Object"), ExecuteReserved.THIS, true, true); + + // Input map of variables passed to the script. + addVariable(null, Definition.getType("Map"), ExecuteReserved.PARAMS, true, true); + + // Scorer parameter passed to the script. Internal use only. + addVariable(null, Definition.DEF_TYPE, ExecuteReserved.SCORER, true, true); + + // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better? + addVariable(null, Definition.getType("Map"), ExecuteReserved.DOC, true, true); + + // Aggregation _value parameter passed to the script. + addVariable(null, Definition.DEF_TYPE, ExecuteReserved.VALUE, true, true); + + // Shortcut variables. + + // Document's score as a read-only double. + if (reserved.usesScore()) { + addVariable(null, Definition.DOUBLE_TYPE, ExecuteReserved.SCORE, true, true); + } + + // The ctx map set by executable scripts as a read-only map. + if (reserved.usesCtx()) { + addVariable(null, Definition.getType("Map"), ExecuteReserved.CTX, true, true); + } + + // Loop counter to catch infinite loops. Internal use only. + if (reserved.getMaxLoopCounter() > 0) { + addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true); + } + } + + public Locals(FunctionReserved reserved, Locals locals, Type rtnType, List parameters) { + this.reserved = reserved; + this.methods = locals.methods; + this.constants = locals.constants; + this.rtnType = rtnType; + + incrementScope(); + + for (Parameter parameter : parameters) { + addVariable(parameter.location, parameter.type, parameter.name, false, false); + } + + // Loop counter to catch infinite loops. Internal use only. + if (reserved.getMaxLoopCounter() > 0) { + addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true); + } + } + + public int getMaxLoopCounter() { + return reserved.getMaxLoopCounter(); + } + + public Method getMethod(MethodKey key) { + return methods.get(key); + } + + public Type getReturnType() { + return rtnType; + } + + public void incrementScope() { + scopes.push(0); + } + + public void decrementScope() { + int remove = scopes.pop(); + + while (remove > 0) { + Variable variable = variables.pop(); + + // This checks whether or not a variable is used when exiting a local scope. + if (variable.read) { + throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] is never used.")); + } + + --remove; + } + } + + public Variable getVariable(Location location, String name) { + Iterator itr = variables.iterator(); + + while (itr.hasNext()) { + Variable variable = itr.next(); + + if (variable.name.equals(name)) { + return variable; + } + } + + throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined.")); + } + + public boolean isVariable(String name) { + Iterator itr = variables.iterator(); + + while (itr.hasNext()) { + Variable variable = itr.next(); + + if (variable.name.equals(name)) { + return true; + } + } + + return false; + } + + public Variable addVariable(Location location, Type type, String name, boolean readonly, boolean reserved) { + if (!reserved && this.reserved.isReserved(name)) { + throw location.createError(new IllegalArgumentException("Variable [" + name + "] is reserved.")); + } + + if (isVariable(name)) { + throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined.")); + } + + Variable previous = variables.peekFirst(); + int slot = 0; + + if (previous != null) { + slot = previous.slot + previous.type.type.getSize(); + } + + Variable variable = new Variable(location, name, type, slot, readonly); + variables.push(variable); + + int update = scopes.pop() + 1; + scopes.push(update); + + return variable; + } + + /** + * Create a new constant. + * + * @param location the location in the script that is creating it + * @param type the type of the constant + * @param name the name of the constant + * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the + * value of the constant on the stack. Generating the load instruction is managed by the caller. + * @return the constant + */ + public Constant addConstant(Location location, org.objectweb.asm.Type type, String name, Consumer initializer) { + if (constants.containsKey(name)) { + throw location.createError(new IllegalArgumentException("Constant [" + name + "] is already defined.")); + } + + Constant constant = new Constant(location, name, type, initializer); + constants.put(name, constant); + return constant; + } + + /** + * Create a new constant. + * + * @param location the location in the script that is creating it + * @param type the type of the constant + * @param name the name of the constant + * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the + * value of the constant on the stack. Generating the load instruction is managed by the caller. + * @return the constant + */ + public Constant addConstant(Location location, Type type, String name, Consumer initializer) { + return addConstant(location, type.type, name, initializer); + } + + public Collection getConstants() { + return constants.values(); + } +} + diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index 7b7caec320e..3280c3bbcfe 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -22,7 +22,6 @@ package org.elasticsearch.painless; import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Type; -import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; @@ -36,15 +35,7 @@ import java.util.Deque; import java.util.List; import static org.elasticsearch.painless.WriterConstants.CHAR_TO_STRING; -import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_DIV_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_LSH_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_MUL_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_OR_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_REM_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_RSH_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_SUB_CALL; +import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE; import static org.elasticsearch.painless.WriterConstants.DEF_TO_BOOLEAN; import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_EXPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_IMPLICIT; @@ -60,9 +51,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_EXPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_IMPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_EXPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT; -import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL; import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE; -import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL; import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS; import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE; @@ -88,27 +77,18 @@ import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE; * shared by the nodes of the Painless tree. */ public final class MethodWriter extends GeneratorAdapter { - private final ClassWriter parent; private final BitSet statements; - private final Deque> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ? - null : new ArrayDeque<>(); + private final Deque> stringConcatArgs = + (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ? null : new ArrayDeque<>(); - MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) { + public MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) { super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null), access, method.getName(), method.getDescriptor()); - this.parent = cw; this.statements = statements; } - /** - * @return A new {@link MethodWriter} with the specified access and signature. - */ - MethodWriter newMethodWriter(int access, Method method) { - return new MethodWriter(access, method, parent, statements); - } - /** * Marks a new statement boundary. *

@@ -282,6 +262,51 @@ public final class MethodWriter extends GeneratorAdapter { } } + /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */ + public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation) { + org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type); + String descriptor = methodType.getDescriptor(); + + switch (operation) { + case MUL: + invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case DIV: + invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case REM: + invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case ADD: + invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case SUB: + invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case LSH: + invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); + break; + case USH: + invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); + break; + case RSH: + invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); + break; + case BWAND: + invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case XOR: + invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + case BWOR: + invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); + break; + default: + throw location.createError(new IllegalStateException("Illegal tree structure.")); + } + } + + /** Writes a static binary instruction */ public void writeBinaryInstruction(Location location, Type type, Operation operation) { final Sort sort = type.sort; @@ -292,38 +317,20 @@ public final class MethodWriter extends GeneratorAdapter { throw location.createError(new IllegalStateException("Illegal tree structure.")); } - if (sort == Sort.DEF) { - switch (operation) { - case MUL: invokeStatic(DEF_UTIL_TYPE, DEF_MUL_CALL); break; - case DIV: invokeStatic(DEF_UTIL_TYPE, DEF_DIV_CALL); break; - case REM: invokeStatic(DEF_UTIL_TYPE, DEF_REM_CALL); break; - case ADD: invokeStatic(DEF_UTIL_TYPE, DEF_ADD_CALL); break; - case SUB: invokeStatic(DEF_UTIL_TYPE, DEF_SUB_CALL); break; - case LSH: invokeStatic(DEF_UTIL_TYPE, DEF_LSH_CALL); break; - case USH: invokeStatic(DEF_UTIL_TYPE, DEF_RSH_CALL); break; - case RSH: invokeStatic(DEF_UTIL_TYPE, DEF_USH_CALL); break; - case BWAND: invokeStatic(DEF_UTIL_TYPE, DEF_AND_CALL); break; - case XOR: invokeStatic(DEF_UTIL_TYPE, DEF_XOR_CALL); break; - case BWOR: invokeStatic(DEF_UTIL_TYPE, DEF_OR_CALL); break; - default: - throw location.createError(new IllegalStateException("Illegal tree structure.")); - } - } else { - switch (operation) { - case MUL: math(GeneratorAdapter.MUL, type.type); break; - case DIV: math(GeneratorAdapter.DIV, type.type); break; - case REM: math(GeneratorAdapter.REM, type.type); break; - case ADD: math(GeneratorAdapter.ADD, type.type); break; - case SUB: math(GeneratorAdapter.SUB, type.type); break; - case LSH: math(GeneratorAdapter.SHL, type.type); break; - case USH: math(GeneratorAdapter.USHR, type.type); break; - case RSH: math(GeneratorAdapter.SHR, type.type); break; - case BWAND: math(GeneratorAdapter.AND, type.type); break; - case XOR: math(GeneratorAdapter.XOR, type.type); break; - case BWOR: math(GeneratorAdapter.OR, type.type); break; - default: - throw location.createError(new IllegalStateException("Illegal tree structure.")); - } + switch (operation) { + case MUL: math(GeneratorAdapter.MUL, type.type); break; + case DIV: math(GeneratorAdapter.DIV, type.type); break; + case REM: math(GeneratorAdapter.REM, type.type); break; + case ADD: math(GeneratorAdapter.ADD, type.type); break; + case SUB: math(GeneratorAdapter.SUB, type.type); break; + case LSH: math(GeneratorAdapter.SHL, type.type); break; + case USH: math(GeneratorAdapter.USHR, type.type); break; + case RSH: math(GeneratorAdapter.SHR, type.type); break; + case BWAND: math(GeneratorAdapter.AND, type.type); break; + case XOR: math(GeneratorAdapter.XOR, type.type); break; + case BWOR: math(GeneratorAdapter.OR, type.type); break; + default: + throw location.createError(new IllegalStateException("Illegal tree structure.")); } } @@ -356,11 +363,16 @@ public final class MethodWriter extends GeneratorAdapter { } @Override - public void visitEnd() { + public void endMethod() { if (stringConcatArgs != null && !stringConcatArgs.isEmpty()) { throw new IllegalStateException("String concat bytecode not completed."); } - super.visitEnd(); + super.endMethod(); + } + + @Override + public void visitEnd() { + throw new AssertionError("Should never call this method on MethodWriter, use endMethod() instead"); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java index 8fdc48e19ba..b303089b339 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java @@ -119,7 +119,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript { public Object run() { try { return executable.execute(variables, scorer, doc, aggregationValue); - } catch (PainlessError | BootstrapMethodError | Exception t) { + } catch (PainlessError | BootstrapMethodError | IllegalAccessError | Exception t) { throw convertToScriptException(t); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java index 5ab3450db7e..b965f25bf0e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java @@ -37,13 +37,5 @@ public class Utility { return value.charAt(0); } - public static boolean checkEquals(final Object left, final Object right) { - if (left != null) { - return left.equals(right); - } - - return right == null || right.equals(null); - } - private Utility() {} } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java deleted file mode 100644 index 4905011520a..00000000000 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.painless; - -import org.elasticsearch.painless.Definition.Type; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Iterator; - -/** - * Tracks variables across compilation phases. - */ -public final class Variables { - - /** - * Tracks reserved variables. Must be given to any source of input - * prior to beginning the analysis phase so that reserved variables - * are known ahead of time to assign appropriate slots without - * being wasteful. - */ - public static final class Reserved { - public static final String THIS = "#this"; - public static final String PARAMS = "params"; - public static final String SCORER = "#scorer"; - public static final String DOC = "doc"; - public static final String VALUE = "_value"; - public static final String SCORE = "_score"; - public static final String CTX = "ctx"; - public static final String LOOP = "#loop"; - - boolean score = false; - boolean ctx = false; - boolean loop = false; - - public void markReserved(String name) { - if (SCORE.equals(name)) { - score = true; - } else if (CTX.equals(name)) { - ctx = true; - } - } - - public boolean isReserved(String name) { - return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) || - name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP); - } - - public void usesLoop() { - loop = true; - } - } - - public static final class Variable { - public final Location location; - public final String name; - public final Type type; - public final int slot; - public final boolean readonly; - - public boolean read = false; - - private Variable(Location location, String name, Type type, int slot, boolean readonly) { - this.location = location; - this.name = name; - this.type = type; - this.slot = slot; - this.readonly = readonly; - } - } - - final Reserved reserved; - - // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead? - private final Deque scopes = new ArrayDeque<>(); - private final Deque variables = new ArrayDeque<>(); - - public Variables(Reserved reserved) { - this.reserved = reserved; - - incrementScope(); - - // Method variables. - - // This reference. Internal use only. - addVariable(null, Definition.getType("Object"), Reserved.THIS, true, true); - - // Input map of variables passed to the script. - addVariable(null, Definition.getType("Map"), Reserved.PARAMS, true, true); - - // Scorer parameter passed to the script. Internal use only. - addVariable(null, Definition.DEF_TYPE, Reserved.SCORER, true, true); - - // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better? - addVariable(null, Definition.getType("Map"), Reserved.DOC, true, true); - - // Aggregation _value parameter passed to the script. - addVariable(null, Definition.DEF_TYPE, Reserved.VALUE, true, true); - - // Shortcut variables. - - // Document's score as a read-only double. - if (reserved.score) { - addVariable(null, Definition.DOUBLE_TYPE, Reserved.SCORE, true, true); - } - - // The ctx map set by executable scripts as a read-only map. - if (reserved.ctx) { - addVariable(null, Definition.getType("Map"), Reserved.CTX, true, true); - } - - // Loop counter to catch infinite loops. Internal use only. - if (reserved.loop) { - addVariable(null, Definition.INT_TYPE, Reserved.LOOP, true, true); - } - } - - public void incrementScope() { - scopes.push(0); - } - - public void decrementScope() { - int remove = scopes.pop(); - - while (remove > 0) { - Variable variable = variables.pop(); - - // TODO: is this working? the code reads backwards... - if (variable.read) { - throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] never used.")); - } - - --remove; - } - } - - public Variable getVariable(Location location, String name) { - Iterator itr = variables.iterator(); - - while (itr.hasNext()) { - Variable variable = itr.next(); - - if (variable.name.equals(name)) { - return variable; - } - } - - throw location.createError(new IllegalArgumentException("Variable [" + name + "] not defined.")); - } - - private boolean variableExists(String name) { - return variables.contains(name); - } - - public Variable addVariable(Location location, Type type, String name, boolean readonly, boolean reserved) { - if (!reserved && this.reserved.isReserved(name)) { - throw location.createError(new IllegalArgumentException("Variable name [" + name + "] is reserved.")); - } - - if (variableExists(name)) { - throw new IllegalArgumentException("Variable name [" + name + "] already defined."); - } - - try { - Definition.getType(name); - } catch (IllegalArgumentException exception) { - // Do nothing. - } - - Variable previous = variables.peekFirst(); - int slot = 0; - - if (previous != null) { - slot = previous.slot + previous.type.type.getSize(); - } - - Variable variable = new Variable(location, name, type, slot, readonly); - variables.push(variable); - - int update = scopes.pop() + 1; - scopes.push(update); - - return variable; - } -} - diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java deleted file mode 100644 index 4d4a778d558..00000000000 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.painless; - -import org.elasticsearch.painless.Variables.Reserved; -import org.elasticsearch.painless.Variables.Variable; -import org.elasticsearch.painless.node.SSource; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.Opcodes; -import org.objectweb.asm.commons.GeneratorAdapter; - -import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE; -import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; -import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR; -import static org.elasticsearch.painless.WriterConstants.EXECUTE; -import static org.elasticsearch.painless.WriterConstants.MAP_GET; -import static org.elasticsearch.painless.WriterConstants.MAP_TYPE; - -import java.util.BitSet; - -/** - * Runs the writing phase of compilation using the Painless AST. - */ -final class Writer { - - static byte[] write(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) { - return new Writer(settings, name, source, variables, root, expressions).getBytes(); - } - - private final CompilerSettings settings; - private final String scriptName; - private final String source; - private final Variables variables; - private final SSource root; - - private final ClassWriter writer; - private final MethodWriter adapter; - - private Writer(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) { - this.settings = settings; - this.scriptName = name; - this.source = source; - this.variables = variables; - this.root = root; - - writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - - writeBegin(); - writeConstructor(); - - adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions); - - writeExecute(); - writeEnd(); - } - - private void writeBegin() { - final int version = Opcodes.V1_8; - final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL; - final String base = BASE_CLASS_TYPE.getInternalName(); - final String name = CLASS_TYPE.getInternalName(); - - // apply marker interface NeedsScore if we use the score! - final String interfaces[] = variables.reserved.score ? - new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null; - - writer.visit(version, access, name, null, base, interfaces); - writer.visitSource(Location.computeSourceName(scriptName,source), null); - } - - private void writeConstructor() { - final GeneratorAdapter constructor = new GeneratorAdapter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, null, null, writer); - constructor.loadThis(); - constructor.loadArgs(); - constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR); - constructor.returnValue(); - constructor.endMethod(); - } - - private void writeExecute() { - if (variables.reserved.score) { - // if the _score value is used, we do this once: - // final double _score = scorer.score(); - final Variable scorer = variables.getVariable(null, Reserved.SCORER); - final Variable score = variables.getVariable(null, Reserved.SCORE); - - adapter.visitVarInsn(Opcodes.ALOAD, scorer.slot); - adapter.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE); - adapter.visitInsn(Opcodes.F2D); - adapter.visitVarInsn(Opcodes.DSTORE, score.slot); - } - - if (variables.reserved.ctx) { - // if the _ctx value is used, we do this once: - // final Map ctx = input.get("ctx"); - - final Variable input = variables.getVariable(null, Reserved.PARAMS); - final Variable ctx = variables.getVariable(null, Reserved.CTX); - - adapter.visitVarInsn(Opcodes.ALOAD, input.slot); - adapter.push(Reserved.CTX); - adapter.invokeInterface(MAP_TYPE, MAP_GET); - adapter.visitVarInsn(Opcodes.ASTORE, ctx.slot); - } - - if (variables.reserved.loop) { - // if there is infinite loop protection, we do this once: - // int #loop = settings.getMaxLoopCounter() - - final Variable loop = variables.getVariable(null, Reserved.LOOP); - - adapter.push(settings.getMaxLoopCounter()); - adapter.visitVarInsn(Opcodes.ISTORE, loop.slot); - } - - root.write(adapter); - adapter.endMethod(); - } - - private void writeEnd() { - writer.visitEnd(); - } - - private byte[] getBytes() { - return writer.toByteArray(); - } -} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 80082c4487b..68814a2e1a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -28,11 +28,14 @@ import org.objectweb.asm.commons.Method; import java.lang.invoke.CallSite; import java.lang.invoke.LambdaMetafactory; +import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.util.BitSet; import java.util.Iterator; import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; /** * General pool of constants used during the writing phase of compilation. @@ -46,6 +49,7 @@ public final class WriterConstants { public final static Type CLASS_TYPE = Type.getObjectType(CLASS_NAME.replace('.', '/')); public final static Method CONSTRUCTOR = getAsmMethod(void.class, "", String.class, String.class, BitSet.class); + public final static Method CLINIT = getAsmMethod(void.class, ""); public final static Method EXECUTE = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class); @@ -65,13 +69,23 @@ public final class WriterConstants { public final static Type UTILITY_TYPE = Type.getType(Utility.class); public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class); public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class); + + public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class); + + /** + * A Method instance for {@linkplain Pattern#compile}. This isn't looked up from Definition because we intentionally don't add it there + * so that the script can't create regexes without this syntax. Essentially, our static regex syntax has a monopoly on building regexes + * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex + * per time it is run. + */ + public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class); /** dynamic callsite bootstrap signature */ public final static MethodType DEF_BOOTSTRAP_TYPE = - MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, long.class); + MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, Object[].class); public final static Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class), - "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString()); + "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); public final static Type DEF_UTIL_TYPE = Type.getType(Def.class); public final static Method DEF_TO_BOOLEAN = getAsmMethod(boolean.class, "DefToboolean" , Object.class); @@ -89,33 +103,15 @@ public final class WriterConstants { public final static Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); public final static Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); - public final static Method DEF_NOT_CALL = getAsmMethod(Object.class , "not", Object.class); - public final static Method DEF_NEG_CALL = getAsmMethod(Object.class , "neg", Object.class); - public final static Method DEF_MUL_CALL = getAsmMethod(Object.class , "mul", Object.class, Object.class); - public final static Method DEF_DIV_CALL = getAsmMethod(Object.class , "div", Object.class, Object.class); - public final static Method DEF_REM_CALL = getAsmMethod(Object.class , "rem", Object.class, Object.class); - public final static Method DEF_ADD_CALL = getAsmMethod(Object.class , "add", Object.class, Object.class); - public final static Method DEF_SUB_CALL = getAsmMethod(Object.class , "sub", Object.class, Object.class); - public final static Method DEF_LSH_CALL = getAsmMethod(Object.class , "lsh", Object.class, int.class); - public final static Method DEF_RSH_CALL = getAsmMethod(Object.class , "rsh", Object.class, int.class); - public final static Method DEF_USH_CALL = getAsmMethod(Object.class , "ush", Object.class, int.class); - public final static Method DEF_AND_CALL = getAsmMethod(Object.class , "and", Object.class, Object.class); - public final static Method DEF_XOR_CALL = getAsmMethod(Object.class , "xor", Object.class, Object.class); - public final static Method DEF_OR_CALL = getAsmMethod(Object.class , "or" , Object.class, Object.class); - public final static Method DEF_EQ_CALL = getAsmMethod(boolean.class, "eq" , Object.class, Object.class); - public final static Method DEF_LT_CALL = getAsmMethod(boolean.class, "lt" , Object.class, Object.class); - public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class); - public final static Method DEF_GT_CALL = getAsmMethod(boolean.class, "gt" , Object.class, Object.class); - public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class); - + /** invokedynamic bootstrap for lambda expression/method references */ public final static MethodType LAMBDA_BOOTSTRAP_TYPE = - MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, + MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, Object[].class); public final static Handle LAMBDA_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class), - "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString()); - + "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); + /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */ public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; static { @@ -126,7 +122,7 @@ public final class WriterConstants { final MethodType type = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class); // ensure it is there: MethodHandles.publicLookup().findStatic(factory, methodName, type); - bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString()); + bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString(), false); } catch (ReflectiveOperationException e) { // not Java 9 - we set it null, so MethodWriter uses StringBuilder: bs = null; @@ -150,9 +146,10 @@ public final class WriterConstants { public final static Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class); public final static Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString"); - public final static Method CHECKEQUALS = getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class); + public final static Type OBJECTS_TYPE = Type.getType(Objects.class); + public final static Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class); - public static Method getAsmMethod(final Class rtype, final String name, final Class... ptypes) { + private static Method getAsmMethod(final Class rtype, final String name, final Class... ptypes) { return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString()); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java index 99453b648a8..d490426c59c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java @@ -33,6 +33,8 @@ final class ErrorHandlingLexer extends PainlessLexer { ErrorHandlingLexer(CharStream charStream, String sourceName) { super(charStream); this.sourceName = sourceName; + // Replace the TokenFactory with a stashing wrapper so we can do token-level lookbehind for regex detection + _factory = new StashingTokenFactory<>(_factory); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java index 9eb5730abc0..143d2a1ef3c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java @@ -22,14 +22,14 @@ class PainlessLexer extends Lexer { public static final int WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, - BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, - MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, - LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, - BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, - ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, - AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, - STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, - DOTID=75; + BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, + BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, + LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, + XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, + INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, + AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, + INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, + TYPE=75, ID=76, DOTINTEGER=77, DOTID=78; public static final int AFTER_DOT = 1; public static String[] modeNames = { "DEFAULT_MODE", "AFTER_DOT" @@ -38,35 +38,35 @@ class PainlessLexer extends Lexer { public static final String[] ruleNames = { "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", - "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", - "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", - "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", - "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", - "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", - "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", - "ID", "DOTINTEGER", "DOTID" + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", + "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", + "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", + "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", + "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", + "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", + "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID" }; private static final String[] _LITERAL_NAMES = { null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", - "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", - "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", - "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", - "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", - "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, - null, null, null, null, "'true'", "'false'", "'null'" + "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", + "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", + "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", + "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", + "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", + "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'" }; private static final String[] _SYMBOLIC_NAMES = { null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", - "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", - "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", - "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", - "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", - "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", - "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", - "ID", "DOTINTEGER", "DOTID" + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", + "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", + "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", + "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", + "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", + "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", + "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -125,21 +125,39 @@ class PainlessLexer extends Lexer { @Override public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 71: + case 27: + return DIV_sempred((RuleContext)_localctx, predIndex); + case 70: + return REGEX_sempred((RuleContext)_localctx, predIndex); + case 74: return TYPE_sempred((RuleContext)_localctx, predIndex); } return true; } + private boolean DIV_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 0: + return false == SlashStrategy.slashIsRegex(_factory) ; + } + return true; + } + private boolean REGEX_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 1: + return SlashStrategy.slashIsRegex(_factory) ; + } + return true; + } private boolean TYPE_sempred(RuleContext _localctx, int predIndex) { switch (predIndex) { - case 0: + case 2: return Definition.isSimpleType(getText()) ; } return true; } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2M\u020e\b\1\b\1\4"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2P\u0228\b\1\b\1\4"+ "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+ "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+ "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+ @@ -148,180 +166,190 @@ class PainlessLexer extends Lexer { "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+ "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+ "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+ - "I\tI\4J\tJ\4K\tK\4L\tL\3\2\6\2\u009c\n\2\r\2\16\2\u009d\3\2\3\2\3\3\3"+ - "\3\3\3\3\3\7\3\u00a6\n\3\f\3\16\3\u00a9\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00b0"+ - "\n\3\f\3\16\3\u00b3\13\3\3\3\3\3\5\3\u00b7\n\3\3\3\3\3\3\4\3\4\3\5\3\5"+ - "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r"+ - "\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3"+ - "\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3"+ - "\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3"+ - "\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3"+ - "\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3"+ - "\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3"+ - "$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3"+ - "+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62"+ - "\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67"+ - "\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3>\3?"+ - "\3?\3?\3?\3@\3@\3@\3@\3@\3A\3A\6A\u0180\nA\rA\16A\u0181\3A\5A\u0185\n"+ - "A\3B\3B\3B\6B\u018a\nB\rB\16B\u018b\3B\5B\u018f\nB\3C\3C\3C\7C\u0194\n"+ - "C\fC\16C\u0197\13C\5C\u0199\nC\3C\5C\u019c\nC\3D\3D\3D\7D\u01a1\nD\fD"+ - "\16D\u01a4\13D\5D\u01a6\nD\3D\3D\6D\u01aa\nD\rD\16D\u01ab\5D\u01ae\nD"+ - "\3D\3D\5D\u01b2\nD\3D\6D\u01b5\nD\rD\16D\u01b6\5D\u01b9\nD\3D\5D\u01bc"+ - "\nD\3E\3E\3E\3E\3E\3E\7E\u01c4\nE\fE\16E\u01c7\13E\3E\3E\3E\3E\3E\3E\3"+ - "E\7E\u01d0\nE\fE\16E\u01d3\13E\3E\5E\u01d6\nE\3F\3F\3F\3F\3F\3G\3G\3G"+ - "\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\3I\3I\7I\u01ec\nI\fI\16I\u01ef\13I\3I\3"+ - "I\3J\3J\7J\u01f5\nJ\fJ\16J\u01f8\13J\3K\3K\3K\7K\u01fd\nK\fK\16K\u0200"+ - "\13K\5K\u0202\nK\3K\3K\3L\3L\7L\u0208\nL\fL\16L\u020b\13L\3L\3L\6\u00a7"+ - "\u00b1\u01c5\u01d1\2M\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30"+ - "\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33"+ - "\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63"+ - "f\64h\65j\66l\67n8p9r:t;v|?~@\u0080A\u0082B\u0084C\u0086D\u0088E"+ - "\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\4\2\3\21\5\2"+ - "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"+ - "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aa"+ - "c|\6\2\62;C\\aac|\u022b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2"+ - "\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26"+ - "\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2"+ - "\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2"+ - "\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2"+ - "\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2"+ - "\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R"+ - "\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3"+ - "\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2"+ - "\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2"+ - "x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2"+ - "\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2"+ - "\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094"+ - "\3\2\2\2\3\u0096\3\2\2\2\3\u0098\3\2\2\2\4\u009b\3\2\2\2\6\u00b6\3\2\2"+ - "\2\b\u00ba\3\2\2\2\n\u00bc\3\2\2\2\f\u00be\3\2\2\2\16\u00c0\3\2\2\2\20"+ - "\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00ca\3\2\2\2\30\u00cc"+ - "\3\2\2\2\32\u00ce\3\2\2\2\34\u00d1\3\2\2\2\36\u00d6\3\2\2\2 \u00dc\3\2"+ - "\2\2\"\u00df\3\2\2\2$\u00e3\3\2\2\2&\u00ec\3\2\2\2(\u00f2\3\2\2\2*\u00f9"+ - "\3\2\2\2,\u00fd\3\2\2\2.\u0101\3\2\2\2\60\u0107\3\2\2\2\62\u010d\3\2\2"+ - "\2\64\u010f\3\2\2\2\66\u0111\3\2\2\28\u0113\3\2\2\2:\u0115\3\2\2\2<\u0117"+ - "\3\2\2\2>\u0119\3\2\2\2@\u011b\3\2\2\2B\u011e\3\2\2\2D\u0121\3\2\2\2F"+ - "\u0125\3\2\2\2H\u0127\3\2\2\2J\u012a\3\2\2\2L\u012c\3\2\2\2N\u012f\3\2"+ - "\2\2P\u0132\3\2\2\2R\u0136\3\2\2\2T\u0139\3\2\2\2V\u013d\3\2\2\2X\u013f"+ - "\3\2\2\2Z\u0141\3\2\2\2\\\u0143\3\2\2\2^\u0146\3\2\2\2`\u0149\3\2\2\2"+ - "b\u014b\3\2\2\2d\u014d\3\2\2\2f\u0150\3\2\2\2h\u0153\3\2\2\2j\u0156\3"+ - "\2\2\2l\u0158\3\2\2\2n\u015b\3\2\2\2p\u015e\3\2\2\2r\u0161\3\2\2\2t\u0164"+ - "\3\2\2\2v\u0167\3\2\2\2x\u016a\3\2\2\2z\u016d\3\2\2\2|\u0170\3\2\2\2~"+ - "\u0174\3\2\2\2\u0080\u0178\3\2\2\2\u0082\u017d\3\2\2\2\u0084\u0186\3\2"+ - "\2\2\u0086\u0198\3\2\2\2\u0088\u01a5\3\2\2\2\u008a\u01d5\3\2\2\2\u008c"+ - "\u01d7\3\2\2\2\u008e\u01dc\3\2\2\2\u0090\u01e2\3\2\2\2\u0092\u01e7\3\2"+ - "\2\2\u0094\u01f2\3\2\2\2\u0096\u0201\3\2\2\2\u0098\u0205\3\2\2\2\u009a"+ - "\u009c\t\2\2\2\u009b\u009a\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009b\3\2"+ - "\2\2\u009d\u009e\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a0\b\2\2\2\u00a0"+ - "\5\3\2\2\2\u00a1\u00a2\7\61\2\2\u00a2\u00a3\7\61\2\2\u00a3\u00a7\3\2\2"+ - "\2\u00a4\u00a6\13\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a9\3\2\2\2\u00a7"+ - "\u00a8\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00aa\3\2\2\2\u00a9\u00a7\3\2"+ - "\2\2\u00aa\u00b7\t\3\2\2\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7,\2\2\u00ad"+ - "\u00b1\3\2\2\2\u00ae\u00b0\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3"+ - "\2\2\2\u00b1\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3"+ - "\u00b1\3\2\2\2\u00b4\u00b5\7,\2\2\u00b5\u00b7\7\61\2\2\u00b6\u00a1\3\2"+ - "\2\2\u00b6\u00ab\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b8\u00b9\b\3\2\2\u00b9"+ - "\7\3\2\2\2\u00ba\u00bb\7}\2\2\u00bb\t\3\2\2\2\u00bc\u00bd\7\177\2\2\u00bd"+ - "\13\3\2\2\2\u00be\u00bf\7]\2\2\u00bf\r\3\2\2\2\u00c0\u00c1\7_\2\2\u00c1"+ - "\17\3\2\2\2\u00c2\u00c3\7*\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\7+\2\2\u00c5"+ - "\23\3\2\2\2\u00c6\u00c7\7\60\2\2\u00c7\u00c8\3\2\2\2\u00c8\u00c9\b\n\3"+ - "\2\u00c9\25\3\2\2\2\u00ca\u00cb\7.\2\2\u00cb\27\3\2\2\2\u00cc\u00cd\7"+ - "=\2\2\u00cd\31\3\2\2\2\u00ce\u00cf\7k\2\2\u00cf\u00d0\7h\2\2\u00d0\33"+ - "\3\2\2\2\u00d1\u00d2\7g\2\2\u00d2\u00d3\7n\2\2\u00d3\u00d4\7u\2\2\u00d4"+ - "\u00d5\7g\2\2\u00d5\35\3\2\2\2\u00d6\u00d7\7y\2\2\u00d7\u00d8\7j\2\2\u00d8"+ - "\u00d9\7k\2\2\u00d9\u00da\7n\2\2\u00da\u00db\7g\2\2\u00db\37\3\2\2\2\u00dc"+ - "\u00dd\7f\2\2\u00dd\u00de\7q\2\2\u00de!\3\2\2\2\u00df\u00e0\7h\2\2\u00e0"+ - "\u00e1\7q\2\2\u00e1\u00e2\7t\2\2\u00e2#\3\2\2\2\u00e3\u00e4\7e\2\2\u00e4"+ - "\u00e5\7q\2\2\u00e5\u00e6\7p\2\2\u00e6\u00e7\7v\2\2\u00e7\u00e8\7k\2\2"+ - "\u00e8\u00e9\7p\2\2\u00e9\u00ea\7w\2\2\u00ea\u00eb\7g\2\2\u00eb%\3\2\2"+ - "\2\u00ec\u00ed\7d\2\2\u00ed\u00ee\7t\2\2\u00ee\u00ef\7g\2\2\u00ef\u00f0"+ - "\7c\2\2\u00f0\u00f1\7m\2\2\u00f1\'\3\2\2\2\u00f2\u00f3\7t\2\2\u00f3\u00f4"+ - "\7g\2\2\u00f4\u00f5\7v\2\2\u00f5\u00f6\7w\2\2\u00f6\u00f7\7t\2\2\u00f7"+ - "\u00f8\7p\2\2\u00f8)\3\2\2\2\u00f9\u00fa\7p\2\2\u00fa\u00fb\7g\2\2\u00fb"+ - "\u00fc\7y\2\2\u00fc+\3\2\2\2\u00fd\u00fe\7v\2\2\u00fe\u00ff\7t\2\2\u00ff"+ - "\u0100\7{\2\2\u0100-\3\2\2\2\u0101\u0102\7e\2\2\u0102\u0103\7c\2\2\u0103"+ - "\u0104\7v\2\2\u0104\u0105\7e\2\2\u0105\u0106\7j\2\2\u0106/\3\2\2\2\u0107"+ - "\u0108\7v\2\2\u0108\u0109\7j\2\2\u0109\u010a\7t\2\2\u010a\u010b\7q\2\2"+ - "\u010b\u010c\7y\2\2\u010c\61\3\2\2\2\u010d\u010e\7#\2\2\u010e\63\3\2\2"+ - "\2\u010f\u0110\7\u0080\2\2\u0110\65\3\2\2\2\u0111\u0112\7,\2\2\u0112\67"+ - "\3\2\2\2\u0113\u0114\7\61\2\2\u01149\3\2\2\2\u0115\u0116\7\'\2\2\u0116"+ - ";\3\2\2\2\u0117\u0118\7-\2\2\u0118=\3\2\2\2\u0119\u011a\7/\2\2\u011a?"+ - "\3\2\2\2\u011b\u011c\7>\2\2\u011c\u011d\7>\2\2\u011dA\3\2\2\2\u011e\u011f"+ - "\7@\2\2\u011f\u0120\7@\2\2\u0120C\3\2\2\2\u0121\u0122\7@\2\2\u0122\u0123"+ - "\7@\2\2\u0123\u0124\7@\2\2\u0124E\3\2\2\2\u0125\u0126\7>\2\2\u0126G\3"+ - "\2\2\2\u0127\u0128\7>\2\2\u0128\u0129\7?\2\2\u0129I\3\2\2\2\u012a\u012b"+ - "\7@\2\2\u012bK\3\2\2\2\u012c\u012d\7@\2\2\u012d\u012e\7?\2\2\u012eM\3"+ - "\2\2\2\u012f\u0130\7?\2\2\u0130\u0131\7?\2\2\u0131O\3\2\2\2\u0132\u0133"+ - "\7?\2\2\u0133\u0134\7?\2\2\u0134\u0135\7?\2\2\u0135Q\3\2\2\2\u0136\u0137"+ - "\7#\2\2\u0137\u0138\7?\2\2\u0138S\3\2\2\2\u0139\u013a\7#\2\2\u013a\u013b"+ - "\7?\2\2\u013b\u013c\7?\2\2\u013cU\3\2\2\2\u013d\u013e\7(\2\2\u013eW\3"+ - "\2\2\2\u013f\u0140\7`\2\2\u0140Y\3\2\2\2\u0141\u0142\7~\2\2\u0142[\3\2"+ - "\2\2\u0143\u0144\7(\2\2\u0144\u0145\7(\2\2\u0145]\3\2\2\2\u0146\u0147"+ - "\7~\2\2\u0147\u0148\7~\2\2\u0148_\3\2\2\2\u0149\u014a\7A\2\2\u014aa\3"+ - "\2\2\2\u014b\u014c\7<\2\2\u014cc\3\2\2\2\u014d\u014e\7<\2\2\u014e\u014f"+ - "\7<\2\2\u014fe\3\2\2\2\u0150\u0151\7-\2\2\u0151\u0152\7-\2\2\u0152g\3"+ - "\2\2\2\u0153\u0154\7/\2\2\u0154\u0155\7/\2\2\u0155i\3\2\2\2\u0156\u0157"+ - "\7?\2\2\u0157k\3\2\2\2\u0158\u0159\7-\2\2\u0159\u015a\7?\2\2\u015am\3"+ - "\2\2\2\u015b\u015c\7/\2\2\u015c\u015d\7?\2\2\u015do\3\2\2\2\u015e\u015f"+ - "\7,\2\2\u015f\u0160\7?\2\2\u0160q\3\2\2\2\u0161\u0162\7\61\2\2\u0162\u0163"+ - "\7?\2\2\u0163s\3\2\2\2\u0164\u0165\7\'\2\2\u0165\u0166\7?\2\2\u0166u\3"+ - "\2\2\2\u0167\u0168\7(\2\2\u0168\u0169\7?\2\2\u0169w\3\2\2\2\u016a\u016b"+ - "\7`\2\2\u016b\u016c\7?\2\2\u016cy\3\2\2\2\u016d\u016e\7~\2\2\u016e\u016f"+ - "\7?\2\2\u016f{\3\2\2\2\u0170\u0171\7>\2\2\u0171\u0172\7>\2\2\u0172\u0173"+ - "\7?\2\2\u0173}\3\2\2\2\u0174\u0175\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177"+ - "\7?\2\2\u0177\177\3\2\2\2\u0178\u0179\7@\2\2\u0179\u017a\7@\2\2\u017a"+ - "\u017b\7@\2\2\u017b\u017c\7?\2\2\u017c\u0081\3\2\2\2\u017d\u017f\7\62"+ - "\2\2\u017e\u0180\t\4\2\2\u017f\u017e\3\2\2\2\u0180\u0181\3\2\2\2\u0181"+ - "\u017f\3\2\2\2\u0181\u0182\3\2\2\2\u0182\u0184\3\2\2\2\u0183\u0185\t\5"+ - "\2\2\u0184\u0183\3\2\2\2\u0184\u0185\3\2\2\2\u0185\u0083\3\2\2\2\u0186"+ - "\u0187\7\62\2\2\u0187\u0189\t\6\2\2\u0188\u018a\t\7\2\2\u0189\u0188\3"+ - "\2\2\2\u018a\u018b\3\2\2\2\u018b\u0189\3\2\2\2\u018b\u018c\3\2\2\2\u018c"+ - "\u018e\3\2\2\2\u018d\u018f\t\5\2\2\u018e\u018d\3\2\2\2\u018e\u018f\3\2"+ - "\2\2\u018f\u0085\3\2\2\2\u0190\u0199\7\62\2\2\u0191\u0195\t\b\2\2\u0192"+ - "\u0194\t\t\2\2\u0193\u0192\3\2\2\2\u0194\u0197\3\2\2\2\u0195\u0193\3\2"+ - "\2\2\u0195\u0196\3\2\2\2\u0196\u0199\3\2\2\2\u0197\u0195\3\2\2\2\u0198"+ - "\u0190\3\2\2\2\u0198\u0191\3\2\2\2\u0199\u019b\3\2\2\2\u019a\u019c\t\n"+ - "\2\2\u019b\u019a\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u0087\3\2\2\2\u019d"+ - "\u01a6\7\62\2\2\u019e\u01a2\t\b\2\2\u019f\u01a1\t\t\2\2\u01a0\u019f\3"+ - "\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+ - "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u019e\3\2"+ - "\2\2\u01a6\u01ad\3\2\2\2\u01a7\u01a9\5\24\n\2\u01a8\u01aa\t\t\2\2\u01a9"+ - "\u01a8\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01a9\3\2\2\2\u01ab\u01ac\3\2"+ - "\2\2\u01ac\u01ae\3\2\2\2\u01ad\u01a7\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae"+ - "\u01b8\3\2\2\2\u01af\u01b1\t\13\2\2\u01b0\u01b2\t\f\2\2\u01b1\u01b0\3"+ - "\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b4\3\2\2\2\u01b3\u01b5\t\t\2\2\u01b4"+ - "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b4\3\2\2\2\u01b6\u01b7\3\2"+ - "\2\2\u01b7\u01b9\3\2\2\2\u01b8\u01af\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9"+ - "\u01bb\3\2\2\2\u01ba\u01bc\t\r\2\2\u01bb\u01ba\3\2\2\2\u01bb\u01bc\3\2"+ - "\2\2\u01bc\u0089\3\2\2\2\u01bd\u01c5\7$\2\2\u01be\u01bf\7^\2\2\u01bf\u01c4"+ - "\7$\2\2\u01c0\u01c1\7^\2\2\u01c1\u01c4\7^\2\2\u01c2\u01c4\n\16\2\2\u01c3"+ - "\u01be\3\2\2\2\u01c3\u01c0\3\2\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c7\3\2"+ - "\2\2\u01c5\u01c6\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7"+ - "\u01c5\3\2\2\2\u01c8\u01d6\7$\2\2\u01c9\u01d1\7)\2\2\u01ca\u01cb\7^\2"+ - "\2\u01cb\u01d0\7)\2\2\u01cc\u01cd\7^\2\2\u01cd\u01d0\7^\2\2\u01ce\u01d0"+ - "\n\16\2\2\u01cf\u01ca\3\2\2\2\u01cf\u01cc\3\2\2\2\u01cf\u01ce\3\2\2\2"+ - "\u01d0\u01d3\3\2\2\2\u01d1\u01d2\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d2\u01d4"+ - "\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01d6\7)\2\2\u01d5\u01bd\3\2\2\2\u01d5"+ - "\u01c9\3\2\2\2\u01d6\u008b\3\2\2\2\u01d7\u01d8\7v\2\2\u01d8\u01d9\7t\2"+ - "\2\u01d9\u01da\7w\2\2\u01da\u01db\7g\2\2\u01db\u008d\3\2\2\2\u01dc\u01dd"+ - "\7h\2\2\u01dd\u01de\7c\2\2\u01de\u01df\7n\2\2\u01df\u01e0\7u\2\2\u01e0"+ - "\u01e1\7g\2\2\u01e1\u008f\3\2\2\2\u01e2\u01e3\7p\2\2\u01e3\u01e4\7w\2"+ - "\2\u01e4\u01e5\7n\2\2\u01e5\u01e6\7n\2\2\u01e6\u0091\3\2\2\2\u01e7\u01ed"+ - "\5\u0094J\2\u01e8\u01e9\5\24\n\2\u01e9\u01ea\5\u0094J\2\u01ea\u01ec\3"+ - "\2\2\2\u01eb\u01e8\3\2\2\2\u01ec\u01ef\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ed"+ - "\u01ee\3\2\2\2\u01ee\u01f0\3\2\2\2\u01ef\u01ed\3\2\2\2\u01f0\u01f1\6I"+ - "\2\2\u01f1\u0093\3\2\2\2\u01f2\u01f6\t\17\2\2\u01f3\u01f5\t\20\2\2\u01f4"+ - "\u01f3\3\2\2\2\u01f5\u01f8\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2"+ - "\2\2\u01f7\u0095\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u0202\7\62\2\2\u01fa"+ - "\u01fe\t\b\2\2\u01fb\u01fd\t\t\2\2\u01fc\u01fb\3\2\2\2\u01fd\u0200\3\2"+ - "\2\2\u01fe\u01fc\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0202\3\2\2\2\u0200"+ - "\u01fe\3\2\2\2\u0201\u01f9\3\2\2\2\u0201\u01fa\3\2\2\2\u0202\u0203\3\2"+ - "\2\2\u0203\u0204\bK\4\2\u0204\u0097\3\2\2\2\u0205\u0209\t\17\2\2\u0206"+ - "\u0208\t\20\2\2\u0207\u0206\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u0207\3"+ - "\2\2\2\u0209\u020a\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c"+ - "\u020d\bL\4\2\u020d\u0099\3\2\2\2!\2\3\u009d\u00a7\u00b1\u00b6\u0181\u0184"+ - "\u018b\u018e\u0195\u0198\u019b\u01a2\u01a5\u01ab\u01ad\u01b1\u01b6\u01b8"+ - "\u01bb\u01c3\u01c5\u01cf\u01d1\u01d5\u01ed\u01f6\u01fe\u0201\u0209\5\b"+ - "\2\2\4\3\2\4\2\2"; + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\3\2\6\2\u00a2\n\2\r\2\16\2\u00a3"+ + "\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00ac\n\3\f\3\16\3\u00af\13\3\3\3\3\3\3\3"+ + "\3\3\3\3\7\3\u00b6\n\3\f\3\16\3\u00b9\13\3\3\3\3\3\5\3\u00bd\n\3\3\3\3"+ + "\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+ + "\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+ + "\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22"+ + "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24"+ + "\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+ + "\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31"+ + "\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37\3\37\3 \3"+ + " \3!\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'\3\'\3\'\3"+ + "(\3(\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3/\3/\3/\3"+ + "\60\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3"+ + "\65\3\65\3\66\3\66\3\66\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;"+ + "\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3B\3B\3B"+ + "\3B\3B\3C\3C\6C\u018f\nC\rC\16C\u0190\3C\5C\u0194\nC\3D\3D\3D\6D\u0199"+ + "\nD\rD\16D\u019a\3D\5D\u019e\nD\3E\3E\3E\7E\u01a3\nE\fE\16E\u01a6\13E"+ + "\5E\u01a8\nE\3E\5E\u01ab\nE\3F\3F\3F\7F\u01b0\nF\fF\16F\u01b3\13F\5F\u01b5"+ + "\nF\3F\3F\6F\u01b9\nF\rF\16F\u01ba\5F\u01bd\nF\3F\3F\5F\u01c1\nF\3F\6"+ + "F\u01c4\nF\rF\16F\u01c5\5F\u01c8\nF\3F\5F\u01cb\nF\3G\3G\3G\3G\3G\3G\7"+ + "G\u01d3\nG\fG\16G\u01d6\13G\3G\3G\3G\3G\3G\3G\3G\7G\u01df\nG\fG\16G\u01e2"+ + "\13G\3G\5G\u01e5\nG\3H\3H\3H\3H\6H\u01eb\nH\rH\16H\u01ec\3H\3H\3H\3I\3"+ + "I\3I\3I\3I\3J\3J\3J\3J\3J\3J\3K\3K\3K\3K\3K\3L\3L\3L\3L\7L\u0206\nL\f"+ + "L\16L\u0209\13L\3L\3L\3M\3M\7M\u020f\nM\fM\16M\u0212\13M\3N\3N\3N\7N\u0217"+ + "\nN\fN\16N\u021a\13N\5N\u021c\nN\3N\3N\3O\3O\7O\u0222\nO\fO\16O\u0225"+ + "\13O\3O\3O\6\u00ad\u00b7\u01d4\u01e0\2P\4\3\6\4\b\5\n\6\f\7\16\b\20\t"+ + "\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27."+ + "\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-"+ + "Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v|?~@\u0080A\u0082B"+ + "\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096"+ + "L\u0098M\u009aN\u009cO\u009eP\4\2\3\23\5\2\13\f\17\17\"\"\4\2\f\f\17\17"+ + "\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4"+ + "\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\5\2C\\aac|\6\2\62"+ + ";C\\aac|\u0247\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2"+ + "\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2"+ + "\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3"+ + "\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2"+ + "\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3"+ + "\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2"+ + "\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2"+ + "T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3"+ + "\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2"+ + "\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2"+ + "z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084"+ + "\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2"+ + "\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+ + "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\3\u009c\3\2\2\2\3\u009e\3\2\2"+ + "\2\4\u00a1\3\2\2\2\6\u00bc\3\2\2\2\b\u00c0\3\2\2\2\n\u00c2\3\2\2\2\f\u00c4"+ + "\3\2\2\2\16\u00c6\3\2\2\2\20\u00c8\3\2\2\2\22\u00ca\3\2\2\2\24\u00cc\3"+ + "\2\2\2\26\u00d0\3\2\2\2\30\u00d2\3\2\2\2\32\u00d4\3\2\2\2\34\u00d7\3\2"+ + "\2\2\36\u00dc\3\2\2\2 \u00e2\3\2\2\2\"\u00e5\3\2\2\2$\u00e9\3\2\2\2&\u00f2"+ + "\3\2\2\2(\u00f8\3\2\2\2*\u00ff\3\2\2\2,\u0103\3\2\2\2.\u0107\3\2\2\2\60"+ + "\u010d\3\2\2\2\62\u0113\3\2\2\2\64\u0118\3\2\2\2\66\u011a\3\2\2\28\u011c"+ + "\3\2\2\2:\u011e\3\2\2\2<\u0121\3\2\2\2>\u0123\3\2\2\2@\u0125\3\2\2\2B"+ + "\u0127\3\2\2\2D\u012a\3\2\2\2F\u012d\3\2\2\2H\u0131\3\2\2\2J\u0133\3\2"+ + "\2\2L\u0136\3\2\2\2N\u0138\3\2\2\2P\u013b\3\2\2\2R\u013e\3\2\2\2T\u0142"+ + "\3\2\2\2V\u0145\3\2\2\2X\u0149\3\2\2\2Z\u014b\3\2\2\2\\\u014d\3\2\2\2"+ + "^\u014f\3\2\2\2`\u0152\3\2\2\2b\u0155\3\2\2\2d\u0157\3\2\2\2f\u0159\3"+ + "\2\2\2h\u015c\3\2\2\2j\u015f\3\2\2\2l\u0162\3\2\2\2n\u0165\3\2\2\2p\u0167"+ + "\3\2\2\2r\u016a\3\2\2\2t\u016d\3\2\2\2v\u0170\3\2\2\2x\u0173\3\2\2\2z"+ + "\u0176\3\2\2\2|\u0179\3\2\2\2~\u017c\3\2\2\2\u0080\u017f\3\2\2\2\u0082"+ + "\u0183\3\2\2\2\u0084\u0187\3\2\2\2\u0086\u018c\3\2\2\2\u0088\u0195\3\2"+ + "\2\2\u008a\u01a7\3\2\2\2\u008c\u01b4\3\2\2\2\u008e\u01e4\3\2\2\2\u0090"+ + "\u01e6\3\2\2\2\u0092\u01f1\3\2\2\2\u0094\u01f6\3\2\2\2\u0096\u01fc\3\2"+ + "\2\2\u0098\u0201\3\2\2\2\u009a\u020c\3\2\2\2\u009c\u021b\3\2\2\2\u009e"+ + "\u021f\3\2\2\2\u00a0\u00a2\t\2\2\2\u00a1\u00a0\3\2\2\2\u00a2\u00a3\3\2"+ + "\2\2\u00a3\u00a1\3\2\2\2\u00a3\u00a4\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5"+ + "\u00a6\b\2\2\2\u00a6\5\3\2\2\2\u00a7\u00a8\7\61\2\2\u00a8\u00a9\7\61\2"+ + "\2\u00a9\u00ad\3\2\2\2\u00aa\u00ac\13\2\2\2\u00ab\u00aa\3\2\2\2\u00ac"+ + "\u00af\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00b0\3\2"+ + "\2\2\u00af\u00ad\3\2\2\2\u00b0\u00bd\t\3\2\2\u00b1\u00b2\7\61\2\2\u00b2"+ + "\u00b3\7,\2\2\u00b3\u00b7\3\2\2\2\u00b4\u00b6\13\2\2\2\u00b5\u00b4\3\2"+ + "\2\2\u00b6\u00b9\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8"+ + "\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bb\7,\2\2\u00bb\u00bd\7\61"+ + "\2\2\u00bc\u00a7\3\2\2\2\u00bc\u00b1\3\2\2\2\u00bd\u00be\3\2\2\2\u00be"+ + "\u00bf\b\3\2\2\u00bf\7\3\2\2\2\u00c0\u00c1\7}\2\2\u00c1\t\3\2\2\2\u00c2"+ + "\u00c3\7\177\2\2\u00c3\13\3\2\2\2\u00c4\u00c5\7]\2\2\u00c5\r\3\2\2\2\u00c6"+ + "\u00c7\7_\2\2\u00c7\17\3\2\2\2\u00c8\u00c9\7*\2\2\u00c9\21\3\2\2\2\u00ca"+ + "\u00cb\7+\2\2\u00cb\23\3\2\2\2\u00cc\u00cd\7\60\2\2\u00cd\u00ce\3\2\2"+ + "\2\u00ce\u00cf\b\n\3\2\u00cf\25\3\2\2\2\u00d0\u00d1\7.\2\2\u00d1\27\3"+ + "\2\2\2\u00d2\u00d3\7=\2\2\u00d3\31\3\2\2\2\u00d4\u00d5\7k\2\2\u00d5\u00d6"+ + "\7h\2\2\u00d6\33\3\2\2\2\u00d7\u00d8\7g\2\2\u00d8\u00d9\7n\2\2\u00d9\u00da"+ + "\7u\2\2\u00da\u00db\7g\2\2\u00db\35\3\2\2\2\u00dc\u00dd\7y\2\2\u00dd\u00de"+ + "\7j\2\2\u00de\u00df\7k\2\2\u00df\u00e0\7n\2\2\u00e0\u00e1\7g\2\2\u00e1"+ + "\37\3\2\2\2\u00e2\u00e3\7f\2\2\u00e3\u00e4\7q\2\2\u00e4!\3\2\2\2\u00e5"+ + "\u00e6\7h\2\2\u00e6\u00e7\7q\2\2\u00e7\u00e8\7t\2\2\u00e8#\3\2\2\2\u00e9"+ + "\u00ea\7e\2\2\u00ea\u00eb\7q\2\2\u00eb\u00ec\7p\2\2\u00ec\u00ed\7v\2\2"+ + "\u00ed\u00ee\7k\2\2\u00ee\u00ef\7p\2\2\u00ef\u00f0\7w\2\2\u00f0\u00f1"+ + "\7g\2\2\u00f1%\3\2\2\2\u00f2\u00f3\7d\2\2\u00f3\u00f4\7t\2\2\u00f4\u00f5"+ + "\7g\2\2\u00f5\u00f6\7c\2\2\u00f6\u00f7\7m\2\2\u00f7\'\3\2\2\2\u00f8\u00f9"+ + "\7t\2\2\u00f9\u00fa\7g\2\2\u00fa\u00fb\7v\2\2\u00fb\u00fc\7w\2\2\u00fc"+ + "\u00fd\7t\2\2\u00fd\u00fe\7p\2\2\u00fe)\3\2\2\2\u00ff\u0100\7p\2\2\u0100"+ + "\u0101\7g\2\2\u0101\u0102\7y\2\2\u0102+\3\2\2\2\u0103\u0104\7v\2\2\u0104"+ + "\u0105\7t\2\2\u0105\u0106\7{\2\2\u0106-\3\2\2\2\u0107\u0108\7e\2\2\u0108"+ + "\u0109\7c\2\2\u0109\u010a\7v\2\2\u010a\u010b\7e\2\2\u010b\u010c\7j\2\2"+ + "\u010c/\3\2\2\2\u010d\u010e\7v\2\2\u010e\u010f\7j\2\2\u010f\u0110\7t\2"+ + "\2\u0110\u0111\7q\2\2\u0111\u0112\7y\2\2\u0112\61\3\2\2\2\u0113\u0114"+ + "\7v\2\2\u0114\u0115\7j\2\2\u0115\u0116\7k\2\2\u0116\u0117\7u\2\2\u0117"+ + "\63\3\2\2\2\u0118\u0119\7#\2\2\u0119\65\3\2\2\2\u011a\u011b\7\u0080\2"+ + "\2\u011b\67\3\2\2\2\u011c\u011d\7,\2\2\u011d9\3\2\2\2\u011e\u011f\7\61"+ + "\2\2\u011f\u0120\6\35\2\2\u0120;\3\2\2\2\u0121\u0122\7\'\2\2\u0122=\3"+ + "\2\2\2\u0123\u0124\7-\2\2\u0124?\3\2\2\2\u0125\u0126\7/\2\2\u0126A\3\2"+ + "\2\2\u0127\u0128\7>\2\2\u0128\u0129\7>\2\2\u0129C\3\2\2\2\u012a\u012b"+ + "\7@\2\2\u012b\u012c\7@\2\2\u012cE\3\2\2\2\u012d\u012e\7@\2\2\u012e\u012f"+ + "\7@\2\2\u012f\u0130\7@\2\2\u0130G\3\2\2\2\u0131\u0132\7>\2\2\u0132I\3"+ + "\2\2\2\u0133\u0134\7>\2\2\u0134\u0135\7?\2\2\u0135K\3\2\2\2\u0136\u0137"+ + "\7@\2\2\u0137M\3\2\2\2\u0138\u0139\7@\2\2\u0139\u013a\7?\2\2\u013aO\3"+ + "\2\2\2\u013b\u013c\7?\2\2\u013c\u013d\7?\2\2\u013dQ\3\2\2\2\u013e\u013f"+ + "\7?\2\2\u013f\u0140\7?\2\2\u0140\u0141\7?\2\2\u0141S\3\2\2\2\u0142\u0143"+ + "\7#\2\2\u0143\u0144\7?\2\2\u0144U\3\2\2\2\u0145\u0146\7#\2\2\u0146\u0147"+ + "\7?\2\2\u0147\u0148\7?\2\2\u0148W\3\2\2\2\u0149\u014a\7(\2\2\u014aY\3"+ + "\2\2\2\u014b\u014c\7`\2\2\u014c[\3\2\2\2\u014d\u014e\7~\2\2\u014e]\3\2"+ + "\2\2\u014f\u0150\7(\2\2\u0150\u0151\7(\2\2\u0151_\3\2\2\2\u0152\u0153"+ + "\7~\2\2\u0153\u0154\7~\2\2\u0154a\3\2\2\2\u0155\u0156\7A\2\2\u0156c\3"+ + "\2\2\2\u0157\u0158\7<\2\2\u0158e\3\2\2\2\u0159\u015a\7<\2\2\u015a\u015b"+ + "\7<\2\2\u015bg\3\2\2\2\u015c\u015d\7/\2\2\u015d\u015e\7@\2\2\u015ei\3"+ + "\2\2\2\u015f\u0160\7-\2\2\u0160\u0161\7-\2\2\u0161k\3\2\2\2\u0162\u0163"+ + "\7/\2\2\u0163\u0164\7/\2\2\u0164m\3\2\2\2\u0165\u0166\7?\2\2\u0166o\3"+ + "\2\2\2\u0167\u0168\7-\2\2\u0168\u0169\7?\2\2\u0169q\3\2\2\2\u016a\u016b"+ + "\7/\2\2\u016b\u016c\7?\2\2\u016cs\3\2\2\2\u016d\u016e\7,\2\2\u016e\u016f"+ + "\7?\2\2\u016fu\3\2\2\2\u0170\u0171\7\61\2\2\u0171\u0172\7?\2\2\u0172w"+ + "\3\2\2\2\u0173\u0174\7\'\2\2\u0174\u0175\7?\2\2\u0175y\3\2\2\2\u0176\u0177"+ + "\7(\2\2\u0177\u0178\7?\2\2\u0178{\3\2\2\2\u0179\u017a\7`\2\2\u017a\u017b"+ + "\7?\2\2\u017b}\3\2\2\2\u017c\u017d\7~\2\2\u017d\u017e\7?\2\2\u017e\177"+ + "\3\2\2\2\u017f\u0180\7>\2\2\u0180\u0181\7>\2\2\u0181\u0182\7?\2\2\u0182"+ + "\u0081\3\2\2\2\u0183\u0184\7@\2\2\u0184\u0185\7@\2\2\u0185\u0186\7?\2"+ + "\2\u0186\u0083\3\2\2\2\u0187\u0188\7@\2\2\u0188\u0189\7@\2\2\u0189\u018a"+ + "\7@\2\2\u018a\u018b\7?\2\2\u018b\u0085\3\2\2\2\u018c\u018e\7\62\2\2\u018d"+ + "\u018f\t\4\2\2\u018e\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190\u018e\3\2"+ + "\2\2\u0190\u0191\3\2\2\2\u0191\u0193\3\2\2\2\u0192\u0194\t\5\2\2\u0193"+ + "\u0192\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0087\3\2\2\2\u0195\u0196\7\62"+ + "\2\2\u0196\u0198\t\6\2\2\u0197\u0199\t\7\2\2\u0198\u0197\3\2\2\2\u0199"+ + "\u019a\3\2\2\2\u019a\u0198\3\2\2\2\u019a\u019b\3\2\2\2\u019b\u019d\3\2"+ + "\2\2\u019c\u019e\t\5\2\2\u019d\u019c\3\2\2\2\u019d\u019e\3\2\2\2\u019e"+ + "\u0089\3\2\2\2\u019f\u01a8\7\62\2\2\u01a0\u01a4\t\b\2\2\u01a1\u01a3\t"+ + "\t\2\2\u01a2\u01a1\3\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a4"+ + "\u01a5\3\2\2\2\u01a5\u01a8\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a7\u019f\3\2"+ + "\2\2\u01a7\u01a0\3\2\2\2\u01a8\u01aa\3\2\2\2\u01a9\u01ab\t\n\2\2\u01aa"+ + "\u01a9\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u008b\3\2\2\2\u01ac\u01b5\7\62"+ + "\2\2\u01ad\u01b1\t\b\2\2\u01ae\u01b0\t\t\2\2\u01af\u01ae\3\2\2\2\u01b0"+ + "\u01b3\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b5\3\2"+ + "\2\2\u01b3\u01b1\3\2\2\2\u01b4\u01ac\3\2\2\2\u01b4\u01ad\3\2\2\2\u01b5"+ + "\u01bc\3\2\2\2\u01b6\u01b8\5\24\n\2\u01b7\u01b9\t\t\2\2\u01b8\u01b7\3"+ + "\2\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb"+ + "\u01bd\3\2\2\2\u01bc\u01b6\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c7\3\2"+ + "\2\2\u01be\u01c0\t\13\2\2\u01bf\u01c1\t\f\2\2\u01c0\u01bf\3\2\2\2\u01c0"+ + "\u01c1\3\2\2\2\u01c1\u01c3\3\2\2\2\u01c2\u01c4\t\t\2\2\u01c3\u01c2\3\2"+ + "\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c5\u01c6\3\2\2\2\u01c6"+ + "\u01c8\3\2\2\2\u01c7\u01be\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01ca\3\2"+ + "\2\2\u01c9\u01cb\t\r\2\2\u01ca\u01c9\3\2\2\2\u01ca\u01cb\3\2\2\2\u01cb"+ + "\u008d\3\2\2\2\u01cc\u01d4\7$\2\2\u01cd\u01ce\7^\2\2\u01ce\u01d3\7$\2"+ + "\2\u01cf\u01d0\7^\2\2\u01d0\u01d3\7^\2\2\u01d1\u01d3\n\16\2\2\u01d2\u01cd"+ + "\3\2\2\2\u01d2\u01cf\3\2\2\2\u01d2\u01d1\3\2\2\2\u01d3\u01d6\3\2\2\2\u01d4"+ + "\u01d5\3\2\2\2\u01d4\u01d2\3\2\2\2\u01d5\u01d7\3\2\2\2\u01d6\u01d4\3\2"+ + "\2\2\u01d7\u01e5\7$\2\2\u01d8\u01e0\7)\2\2\u01d9\u01da\7^\2\2\u01da\u01df"+ + "\7)\2\2\u01db\u01dc\7^\2\2\u01dc\u01df\7^\2\2\u01dd\u01df\n\16\2\2\u01de"+ + "\u01d9\3\2\2\2\u01de\u01db\3\2\2\2\u01de\u01dd\3\2\2\2\u01df\u01e2\3\2"+ + "\2\2\u01e0\u01e1\3\2\2\2\u01e0\u01de\3\2\2\2\u01e1\u01e3\3\2\2\2\u01e2"+ + "\u01e0\3\2\2\2\u01e3\u01e5\7)\2\2\u01e4\u01cc\3\2\2\2\u01e4\u01d8\3\2"+ + "\2\2\u01e5\u008f\3\2\2\2\u01e6\u01ea\7\61\2\2\u01e7\u01eb\n\17\2\2\u01e8"+ + "\u01e9\7^\2\2\u01e9\u01eb\n\20\2\2\u01ea\u01e7\3\2\2\2\u01ea\u01e8\3\2"+ + "\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ea\3\2\2\2\u01ec\u01ed\3\2\2\2\u01ed"+ + "\u01ee\3\2\2\2\u01ee\u01ef\7\61\2\2\u01ef\u01f0\6H\3\2\u01f0\u0091\3\2"+ + "\2\2\u01f1\u01f2\7v\2\2\u01f2\u01f3\7t\2\2\u01f3\u01f4\7w\2\2\u01f4\u01f5"+ + "\7g\2\2\u01f5\u0093\3\2\2\2\u01f6\u01f7\7h\2\2\u01f7\u01f8\7c\2\2\u01f8"+ + "\u01f9\7n\2\2\u01f9\u01fa\7u\2\2\u01fa\u01fb\7g\2\2\u01fb\u0095\3\2\2"+ + "\2\u01fc\u01fd\7p\2\2\u01fd\u01fe\7w\2\2\u01fe\u01ff\7n\2\2\u01ff\u0200"+ + "\7n\2\2\u0200\u0097\3\2\2\2\u0201\u0207\5\u009aM\2\u0202\u0203\5\24\n"+ + "\2\u0203\u0204\5\u009aM\2\u0204\u0206\3\2\2\2\u0205\u0202\3\2\2\2\u0206"+ + "\u0209\3\2\2\2\u0207\u0205\3\2\2\2\u0207\u0208\3\2\2\2\u0208\u020a\3\2"+ + "\2\2\u0209\u0207\3\2\2\2\u020a\u020b\6L\4\2\u020b\u0099\3\2\2\2\u020c"+ + "\u0210\t\21\2\2\u020d\u020f\t\22\2\2\u020e\u020d\3\2\2\2\u020f\u0212\3"+ + "\2\2\2\u0210\u020e\3\2\2\2\u0210\u0211\3\2\2\2\u0211\u009b\3\2\2\2\u0212"+ + "\u0210\3\2\2\2\u0213\u021c\7\62\2\2\u0214\u0218\t\b\2\2\u0215\u0217\t"+ + "\t\2\2\u0216\u0215\3\2\2\2\u0217\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218"+ + "\u0219\3\2\2\2\u0219\u021c\3\2\2\2\u021a\u0218\3\2\2\2\u021b\u0213\3\2"+ + "\2\2\u021b\u0214\3\2\2\2\u021c\u021d\3\2\2\2\u021d\u021e\bN\4\2\u021e"+ + "\u009d\3\2\2\2\u021f\u0223\t\21\2\2\u0220\u0222\t\22\2\2\u0221\u0220\3"+ + "\2\2\2\u0222\u0225\3\2\2\2\u0223\u0221\3\2\2\2\u0223\u0224\3\2\2\2\u0224"+ + "\u0226\3\2\2\2\u0225\u0223\3\2\2\2\u0226\u0227\bO\4\2\u0227\u009f\3\2"+ + "\2\2#\2\3\u00a3\u00ad\u00b7\u00bc\u0190\u0193\u019a\u019d\u01a4\u01a7"+ + "\u01aa\u01b1\u01b4\u01ba\u01bc\u01c0\u01c5\u01c7\u01ca\u01d2\u01d4\u01de"+ + "\u01e0\u01e4\u01ea\u01ec\u0207\u0210\u0218\u021b\u0223\5\b\2\2\4\3\2\4"+ + "\2\2"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java index 540773fbf70..ae9508f7969 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java @@ -19,48 +19,51 @@ class PainlessParser extends Parser { public static final int WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, - BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, - MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, - LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, - BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, - ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, - AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, - STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, - DOTID=75; + BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, + BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, + LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, + XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, + INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, + AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, + INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, + TYPE=75, ID=76, DOTINTEGER=77, DOTID=78; public static final int - RULE_source = 0, RULE_statement = 1, RULE_trailer = 2, RULE_block = 3, - RULE_empty = 4, RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, - RULE_decltype = 8, RULE_funcref = 9, RULE_declvar = 10, RULE_trap = 11, - RULE_delimiter = 12, RULE_expression = 13, RULE_unary = 14, RULE_chain = 15, - RULE_primary = 16, RULE_secondary = 17, RULE_dot = 18, RULE_brace = 19, - RULE_arguments = 20, RULE_argument = 21; + RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, + RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, + RULE_afterthought = 8, RULE_declaration = 9, RULE_decltype = 10, RULE_declvar = 11, + RULE_trap = 12, RULE_delimiter = 13, RULE_expression = 14, RULE_unary = 15, + RULE_chain = 16, RULE_primary = 17, RULE_secondary = 18, RULE_dot = 19, + RULE_brace = 20, RULE_arguments = 21, RULE_argument = 22, RULE_lambda = 23, + RULE_lamtype = 24, RULE_funcref = 25, RULE_classFuncref = 26, RULE_constructorFuncref = 27, + RULE_capturingFuncref = 28, RULE_localFuncref = 29; public static final String[] ruleNames = { - "source", "statement", "trailer", "block", "empty", "initializer", "afterthought", - "declaration", "decltype", "funcref", "declvar", "trap", "delimiter", - "expression", "unary", "chain", "primary", "secondary", "dot", "brace", - "arguments", "argument" + "source", "function", "parameters", "statement", "trailer", "block", "empty", + "initializer", "afterthought", "declaration", "decltype", "declvar", "trap", + "delimiter", "expression", "unary", "chain", "primary", "secondary", "dot", + "brace", "arguments", "argument", "lambda", "lamtype", "funcref", "classFuncref", + "constructorFuncref", "capturingFuncref", "localFuncref" }; private static final String[] _LITERAL_NAMES = { null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", - "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", - "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", - "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", - "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", - "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, - null, null, null, null, "'true'", "'false'", "'null'" + "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", + "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", + "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", + "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", + "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", + "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'" }; private static final String[] _SYMBOLIC_NAMES = { null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", - "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", - "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", - "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", - "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", - "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", - "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", - "ID", "DOTINTEGER", "DOTID" + "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", + "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", + "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", + "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", + "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", + "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", + "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -113,6 +116,12 @@ class PainlessParser extends Parser { } public static class SourceContext extends ParserRuleContext { public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); } + public List function() { + return getRuleContexts(FunctionContext.class); + } + public FunctionContext function(int i) { + return getRuleContext(FunctionContext.class,i); + } public List statement() { return getRuleContexts(StatementContext.class); } @@ -137,23 +146,39 @@ class PainlessParser extends Parser { int _alt; enterOuterAlt(_localctx, 1); { - setState(47); + setState(63); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(44); + setState(60); + function(); + } + } + } + setState(65); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,0,_ctx); + } + setState(69); + _errHandler.sync(this); + _alt = getInterpreter().adaptivePredict(_input,1,_ctx); + while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { + if ( _alt==1 ) { + { + { + setState(66); statement(); } } } - setState(49); + setState(71); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,0,_ctx); + _alt = getInterpreter().adaptivePredict(_input,1,_ctx); } - setState(50); + setState(72); match(EOF); } } @@ -168,6 +193,136 @@ class PainlessParser extends Parser { return _localctx; } + public static class FunctionContext extends ParserRuleContext { + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } + public ParametersContext parameters() { + return getRuleContext(ParametersContext.class,0); + } + public BlockContext block() { + return getRuleContext(BlockContext.class,0); + } + public FunctionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_function; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitFunction(this); + else return visitor.visitChildren(this); + } + } + + public final FunctionContext function() throws RecognitionException { + FunctionContext _localctx = new FunctionContext(_ctx, getState()); + enterRule(_localctx, 2, RULE_function); + try { + enterOuterAlt(_localctx, 1); + { + setState(74); + decltype(); + setState(75); + match(ID); + setState(76); + parameters(); + setState(77); + block(); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ParametersContext extends ParserRuleContext { + public TerminalNode LP() { return getToken(PainlessParser.LP, 0); } + public TerminalNode RP() { return getToken(PainlessParser.RP, 0); } + public List decltype() { + return getRuleContexts(DecltypeContext.class); + } + public DecltypeContext decltype(int i) { + return getRuleContext(DecltypeContext.class,i); + } + public List ID() { return getTokens(PainlessParser.ID); } + public TerminalNode ID(int i) { + return getToken(PainlessParser.ID, i); + } + public List COMMA() { return getTokens(PainlessParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(PainlessParser.COMMA, i); + } + public ParametersContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_parameters; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitParameters(this); + else return visitor.visitChildren(this); + } + } + + public final ParametersContext parameters() throws RecognitionException { + ParametersContext _localctx = new ParametersContext(_ctx, getState()); + enterRule(_localctx, 4, RULE_parameters); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(79); + match(LP); + setState(91); + _la = _input.LA(1); + if (_la==TYPE) { + { + setState(80); + decltype(); + setState(81); + match(ID); + setState(88); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la==COMMA) { + { + { + setState(82); + match(COMMA); + setState(83); + decltype(); + setState(84); + match(ID); + } + } + setState(90); + _errHandler.sync(this); + _la = _input.LA(1); + } + } + } + + setState(93); + match(RP); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + public static class StatementContext extends ParserRuleContext { public StatementContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); @@ -396,38 +551,38 @@ class PainlessParser extends Parser { public final StatementContext statement() throws RecognitionException { StatementContext _localctx = new StatementContext(_ctx, getState()); - enterRule(_localctx, 2, RULE_statement); + enterRule(_localctx, 6, RULE_statement); try { int _alt; - setState(130); - switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) { + setState(173); + switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: _localctx = new IfContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(52); + setState(95); match(IF); - setState(53); + setState(96); match(LP); - setState(54); + setState(97); expression(0); - setState(55); + setState(98); match(RP); - setState(56); + setState(99); trailer(); - setState(60); - switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { + setState(103); + switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) { case 1: { - setState(57); + setState(100); match(ELSE); - setState(58); + setState(101); trailer(); } break; case 2: { - setState(59); + setState(102); if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE "); } break; @@ -438,25 +593,25 @@ class PainlessParser extends Parser { _localctx = new WhileContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(62); + setState(105); match(WHILE); - setState(63); + setState(106); match(LP); - setState(64); + setState(107); expression(0); - setState(65); + setState(108); match(RP); - setState(68); - switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { + setState(111); + switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { case 1: { - setState(66); + setState(109); trailer(); } break; case 2: { - setState(67); + setState(110); empty(); } break; @@ -467,19 +622,19 @@ class PainlessParser extends Parser { _localctx = new DoContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(70); + setState(113); match(DO); - setState(71); + setState(114); block(); - setState(72); + setState(115); match(WHILE); - setState(73); + setState(116); match(LP); - setState(74); + setState(117); expression(0); - setState(75); + setState(118); match(RP); - setState(76); + setState(119); delimiter(); } break; @@ -487,54 +642,54 @@ class PainlessParser extends Parser { _localctx = new ForContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(78); + setState(121); match(FOR); - setState(79); + setState(122); match(LP); - setState(81); - switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) { + setState(124); + switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: { - setState(80); + setState(123); initializer(); } break; } - setState(83); + setState(126); match(SEMICOLON); - setState(85); - switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) { + setState(128); + switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: { - setState(84); + setState(127); expression(0); } break; } - setState(87); + setState(130); match(SEMICOLON); - setState(89); - switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { + setState(132); + switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) { case 1: { - setState(88); + setState(131); afterthought(); } break; } - setState(91); + setState(134); match(RP); - setState(94); - switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { + setState(137); + switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) { case 1: { - setState(92); + setState(135); trailer(); } break; case 2: { - setState(93); + setState(136); empty(); } break; @@ -545,21 +700,21 @@ class PainlessParser extends Parser { _localctx = new EachContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(96); + setState(139); match(FOR); - setState(97); + setState(140); match(LP); - setState(98); + setState(141); decltype(); - setState(99); + setState(142); match(ID); - setState(100); + setState(143); match(COLON); - setState(101); + setState(144); expression(0); - setState(102); + setState(145); match(RP); - setState(103); + setState(146); trailer(); } break; @@ -567,9 +722,9 @@ class PainlessParser extends Parser { _localctx = new DeclContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(105); + setState(148); declaration(); - setState(106); + setState(149); delimiter(); } break; @@ -577,9 +732,9 @@ class PainlessParser extends Parser { _localctx = new ContinueContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(108); + setState(151); match(CONTINUE); - setState(109); + setState(152); delimiter(); } break; @@ -587,9 +742,9 @@ class PainlessParser extends Parser { _localctx = new BreakContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(110); + setState(153); match(BREAK); - setState(111); + setState(154); delimiter(); } break; @@ -597,11 +752,11 @@ class PainlessParser extends Parser { _localctx = new ReturnContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(112); + setState(155); match(RETURN); - setState(113); + setState(156); expression(0); - setState(114); + setState(157); delimiter(); } break; @@ -609,11 +764,11 @@ class PainlessParser extends Parser { _localctx = new TryContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(116); + setState(159); match(TRY); - setState(117); + setState(160); block(); - setState(119); + setState(162); _errHandler.sync(this); _alt = 1; do { @@ -621,7 +776,7 @@ class PainlessParser extends Parser { case 1: { { - setState(118); + setState(161); trap(); } } @@ -629,9 +784,9 @@ class PainlessParser extends Parser { default: throw new NoViableAltException(this); } - setState(121); + setState(164); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,7,_ctx); + _alt = getInterpreter().adaptivePredict(_input,10,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); } break; @@ -639,11 +794,11 @@ class PainlessParser extends Parser { _localctx = new ThrowContext(_localctx); enterOuterAlt(_localctx, 11); { - setState(123); + setState(166); match(THROW); - setState(124); + setState(167); expression(0); - setState(125); + setState(168); delimiter(); } break; @@ -651,9 +806,9 @@ class PainlessParser extends Parser { _localctx = new ExprContext(_localctx); enterOuterAlt(_localctx, 12); { - setState(127); + setState(170); expression(0); - setState(128); + setState(171); delimiter(); } break; @@ -690,21 +845,21 @@ class PainlessParser extends Parser { public final TrailerContext trailer() throws RecognitionException { TrailerContext _localctx = new TrailerContext(_ctx, getState()); - enterRule(_localctx, 4, RULE_trailer); + enterRule(_localctx, 8, RULE_trailer); try { - setState(134); - switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) { + setState(177); + switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(132); + setState(175); block(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(133); + setState(176); statement(); } break; @@ -743,30 +898,30 @@ class PainlessParser extends Parser { public final BlockContext block() throws RecognitionException { BlockContext _localctx = new BlockContext(_ctx, getState()); - enterRule(_localctx, 6, RULE_block); + enterRule(_localctx, 10, RULE_block); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(136); + setState(179); match(LBRACK); - setState(140); + setState(183); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,10,_ctx); + _alt = getInterpreter().adaptivePredict(_input,13,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(137); + setState(180); statement(); } } } - setState(142); + setState(185); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,10,_ctx); + _alt = getInterpreter().adaptivePredict(_input,13,_ctx); } - setState(143); + setState(186); match(RBRACK); } } @@ -796,11 +951,11 @@ class PainlessParser extends Parser { public final EmptyContext empty() throws RecognitionException { EmptyContext _localctx = new EmptyContext(_ctx, getState()); - enterRule(_localctx, 8, RULE_empty); + enterRule(_localctx, 12, RULE_empty); try { enterOuterAlt(_localctx, 1); { - setState(145); + setState(188); match(SEMICOLON); } } @@ -835,21 +990,21 @@ class PainlessParser extends Parser { public final InitializerContext initializer() throws RecognitionException { InitializerContext _localctx = new InitializerContext(_ctx, getState()); - enterRule(_localctx, 10, RULE_initializer); + enterRule(_localctx, 14, RULE_initializer); try { - setState(149); - switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { + setState(192); + switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(147); + setState(190); declaration(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(148); + setState(191); expression(0); } break; @@ -883,11 +1038,11 @@ class PainlessParser extends Parser { public final AfterthoughtContext afterthought() throws RecognitionException { AfterthoughtContext _localctx = new AfterthoughtContext(_ctx, getState()); - enterRule(_localctx, 12, RULE_afterthought); + enterRule(_localctx, 16, RULE_afterthought); try { enterOuterAlt(_localctx, 1); { - setState(151); + setState(194); expression(0); } } @@ -929,28 +1084,28 @@ class PainlessParser extends Parser { public final DeclarationContext declaration() throws RecognitionException { DeclarationContext _localctx = new DeclarationContext(_ctx, getState()); - enterRule(_localctx, 14, RULE_declaration); + enterRule(_localctx, 18, RULE_declaration); int _la; try { enterOuterAlt(_localctx, 1); { - setState(153); + setState(196); decltype(); - setState(154); + setState(197); declvar(); - setState(159); + setState(202); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(155); + setState(198); match(COMMA); - setState(156); + setState(199); declvar(); } } - setState(161); + setState(204); _errHandler.sync(this); _la = _input.LA(1); } @@ -990,26 +1145,26 @@ class PainlessParser extends Parser { public final DecltypeContext decltype() throws RecognitionException { DecltypeContext _localctx = new DecltypeContext(_ctx, getState()); - enterRule(_localctx, 16, RULE_decltype); + enterRule(_localctx, 20, RULE_decltype); int _la; try { enterOuterAlt(_localctx, 1); { - setState(162); + setState(205); match(TYPE); - setState(167); + setState(210); _errHandler.sync(this); _la = _input.LA(1); while (_la==LBRACE) { { { - setState(163); + setState(206); match(LBRACE); - setState(164); + setState(207); match(RBRACE); } } - setState(169); + setState(212); _errHandler.sync(this); _la = _input.LA(1); } @@ -1026,53 +1181,6 @@ class PainlessParser extends Parser { return _localctx; } - public static class FuncrefContext extends ParserRuleContext { - public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); } - public TerminalNode REF() { return getToken(PainlessParser.REF, 0); } - public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } - public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); } - public FuncrefContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_funcref; } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitFuncref(this); - else return visitor.visitChildren(this); - } - } - - public final FuncrefContext funcref() throws RecognitionException { - FuncrefContext _localctx = new FuncrefContext(_ctx, getState()); - enterRule(_localctx, 18, RULE_funcref); - int _la; - try { - enterOuterAlt(_localctx, 1); - { - setState(170); - match(TYPE); - setState(171); - match(REF); - setState(172); - _la = _input.LA(1); - if ( !(_la==NEW || _la==ID) ) { - _errHandler.recoverInline(this); - } else { - consume(); - } - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - public static class DeclvarContext extends ParserRuleContext { public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } public TerminalNode ASSIGN() { return getToken(PainlessParser.ASSIGN, 0); } @@ -1092,20 +1200,20 @@ class PainlessParser extends Parser { public final DeclvarContext declvar() throws RecognitionException { DeclvarContext _localctx = new DeclvarContext(_ctx, getState()); - enterRule(_localctx, 20, RULE_declvar); + enterRule(_localctx, 22, RULE_declvar); int _la; try { enterOuterAlt(_localctx, 1); { - setState(174); + setState(213); match(ID); - setState(177); + setState(216); _la = _input.LA(1); if (_la==ASSIGN) { { - setState(175); + setState(214); match(ASSIGN); - setState(176); + setState(215); expression(0); } } @@ -1145,21 +1253,21 @@ class PainlessParser extends Parser { public final TrapContext trap() throws RecognitionException { TrapContext _localctx = new TrapContext(_ctx, getState()); - enterRule(_localctx, 22, RULE_trap); + enterRule(_localctx, 24, RULE_trap); try { enterOuterAlt(_localctx, 1); { - setState(179); + setState(218); match(CATCH); - setState(180); + setState(219); match(LP); - setState(181); + setState(220); match(TYPE); - setState(182); + setState(221); match(ID); - setState(183); + setState(222); match(RP); - setState(184); + setState(223); block(); } } @@ -1177,6 +1285,7 @@ class PainlessParser extends Parser { public static class DelimiterContext extends ParserRuleContext { public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); } public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); } + public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); } public DelimiterContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @@ -1190,18 +1299,35 @@ class PainlessParser extends Parser { public final DelimiterContext delimiter() throws RecognitionException { DelimiterContext _localctx = new DelimiterContext(_ctx, getState()); - enterRule(_localctx, 24, RULE_delimiter); - int _la; + enterRule(_localctx, 26, RULE_delimiter); try { - enterOuterAlt(_localctx, 1); - { - setState(186); - _la = _input.LA(1); - if ( !(_la==EOF || _la==SEMICOLON) ) { - _errHandler.recoverInline(this); - } else { - consume(); - } + setState(230); + switch (_input.LA(1)) { + case SEMICOLON: + enterOuterAlt(_localctx, 1); + { + setState(225); + match(SEMICOLON); + } + break; + case EOF: + enterOuterAlt(_localctx, 2); + { + setState(226); + match(EOF); + } + break; + case RBRACK: + enterOuterAlt(_localctx, 3); + { + int mark = _input.mark(); int index = _input.index(); + setState(228); + match(RBRACK); + _input.seek(index); _input.release(mark); + } + break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -1357,31 +1483,31 @@ class PainlessParser extends Parser { int _parentState = getState(); ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState); ExpressionContext _prevctx = _localctx; - int _startState = 26; - enterRecursionRule(_localctx, 26, RULE_expression, _p); + int _startState = 28; + enterRecursionRule(_localctx, 28, RULE_expression, _p); int _la; try { int _alt; enterOuterAlt(_localctx, 1); { - setState(197); - switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { + setState(241); + switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) { case 1: { _localctx = new AssignmentContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(189); + setState(233); chain(true); - setState(190); + setState(234); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) { + if ( !(((((_la - 54)) & ~0x3f) == 0 && ((1L << (_la - 54)) & ((1L << (ASSIGN - 54)) | (1L << (AADD - 54)) | (1L << (ASUB - 54)) | (1L << (AMUL - 54)) | (1L << (ADIV - 54)) | (1L << (AREM - 54)) | (1L << (AAND - 54)) | (1L << (AXOR - 54)) | (1L << (AOR - 54)) | (1L << (ALSH - 54)) | (1L << (ARSH - 54)) | (1L << (AUSH - 54)))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(191); + setState(235); expression(1); ((AssignmentContext)_localctx).s = false; } @@ -1391,37 +1517,37 @@ class PainlessParser extends Parser { _localctx = new SingleContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(194); + setState(238); ((SingleContext)_localctx).u = unary(false); ((SingleContext)_localctx).s = ((SingleContext)_localctx).u.s; } break; } _ctx.stop = _input.LT(-1); - setState(258); + setState(302); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,17,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(256); - switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { + setState(300); + switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(199); + setState(243); if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)"); - setState(200); + setState(244); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(201); + setState(245); expression(13); ((BinaryContext)_localctx).s = false; } @@ -1430,16 +1556,16 @@ class PainlessParser extends Parser { { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(204); + setState(248); if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)"); - setState(205); + setState(249); _la = _input.LA(1); if ( !(_la==ADD || _la==SUB) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(206); + setState(250); expression(12); ((BinaryContext)_localctx).s = false; } @@ -1448,16 +1574,16 @@ class PainlessParser extends Parser { { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(209); + setState(253); if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)"); - setState(210); + setState(254); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(211); + setState(255); expression(11); ((BinaryContext)_localctx).s = false; } @@ -1466,16 +1592,16 @@ class PainlessParser extends Parser { { _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(214); + setState(258); if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)"); - setState(215); + setState(259); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(216); + setState(260); expression(10); ((CompContext)_localctx).s = false; } @@ -1484,16 +1610,16 @@ class PainlessParser extends Parser { { _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(219); + setState(263); if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)"); - setState(220); + setState(264); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(221); + setState(265); expression(9); ((CompContext)_localctx).s = false; } @@ -1502,11 +1628,11 @@ class PainlessParser extends Parser { { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(224); + setState(268); if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)"); - setState(225); + setState(269); match(BWAND); - setState(226); + setState(270); expression(8); ((BinaryContext)_localctx).s = false; } @@ -1515,11 +1641,11 @@ class PainlessParser extends Parser { { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(229); + setState(273); if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)"); - setState(230); + setState(274); match(XOR); - setState(231); + setState(275); expression(7); ((BinaryContext)_localctx).s = false; } @@ -1528,11 +1654,11 @@ class PainlessParser extends Parser { { _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(234); + setState(278); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(235); + setState(279); match(BWOR); - setState(236); + setState(280); expression(6); ((BinaryContext)_localctx).s = false; } @@ -1541,11 +1667,11 @@ class PainlessParser extends Parser { { _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(239); + setState(283); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(240); + setState(284); match(BOOLAND); - setState(241); + setState(285); expression(5); ((BoolContext)_localctx).s = false; } @@ -1554,11 +1680,11 @@ class PainlessParser extends Parser { { _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(244); + setState(288); if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(245); + setState(289); match(BOOLOR); - setState(246); + setState(290); expression(4); ((BoolContext)_localctx).s = false; } @@ -1567,15 +1693,15 @@ class PainlessParser extends Parser { { _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_expression); - setState(249); + setState(293); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(250); + setState(294); match(COND); - setState(251); + setState(295); ((ConditionalContext)_localctx).e0 = expression(0); - setState(252); + setState(296); match(COLON); - setState(253); + setState(297); ((ConditionalContext)_localctx).e1 = expression(2); ((ConditionalContext)_localctx).s = ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; } @@ -1583,9 +1709,9 @@ class PainlessParser extends Parser { } } } - setState(260); + setState(304); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,17,_ctx); + _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } } } @@ -1727,25 +1853,25 @@ class PainlessParser extends Parser { public final UnaryContext unary(boolean c) throws RecognitionException { UnaryContext _localctx = new UnaryContext(_ctx, getState(), c); - enterRule(_localctx, 28, RULE_unary); + enterRule(_localctx, 30, RULE_unary); int _la; try { - setState(290); - switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) { + setState(334); + switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: _localctx = new PreContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(261); + setState(305); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(262); + setState(306); _la = _input.LA(1); if ( !(_la==INCR || _la==DECR) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(263); + setState(307); chain(true); } break; @@ -1753,11 +1879,11 @@ class PainlessParser extends Parser { _localctx = new PostContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(264); + setState(308); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(265); + setState(309); chain(true); - setState(266); + setState(310); _la = _input.LA(1); if ( !(_la==INCR || _la==DECR) ) { _errHandler.recoverInline(this); @@ -1770,9 +1896,9 @@ class PainlessParser extends Parser { _localctx = new ReadContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(268); + setState(312); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(269); + setState(313); chain(false); } break; @@ -1780,11 +1906,11 @@ class PainlessParser extends Parser { _localctx = new NumericContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(270); + setState(314); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(271); + setState(315); _la = _input.LA(1); - if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) { + if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & ((1L << (OCTAL - 66)) | (1L << (HEX - 66)) | (1L << (INTEGER - 66)) | (1L << (DECIMAL - 66)))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); @@ -1796,9 +1922,9 @@ class PainlessParser extends Parser { _localctx = new TrueContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(273); + setState(317); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(274); + setState(318); match(TRUE); ((TrueContext)_localctx).s = false; } @@ -1807,9 +1933,9 @@ class PainlessParser extends Parser { _localctx = new FalseContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(276); + setState(320); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(277); + setState(321); match(FALSE); ((FalseContext)_localctx).s = false; } @@ -1818,9 +1944,9 @@ class PainlessParser extends Parser { _localctx = new NullContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(279); + setState(323); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(280); + setState(324); match(NULL); ((NullContext)_localctx).s = false; } @@ -1829,16 +1955,16 @@ class PainlessParser extends Parser { _localctx = new OperatorContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(282); + setState(326); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(283); + setState(327); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(284); + setState(328); unary(false); } break; @@ -1846,13 +1972,13 @@ class PainlessParser extends Parser { _localctx = new CastContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(285); + setState(329); match(LP); - setState(286); + setState(330); decltype(); - setState(287); + setState(331); match(RP); - setState(288); + setState(332); unary(_localctx.c); } break; @@ -1958,32 +2084,32 @@ class PainlessParser extends Parser { public final ChainContext chain(boolean c) throws RecognitionException { ChainContext _localctx = new ChainContext(_ctx, getState(), c); - enterRule(_localctx, 30, RULE_chain); + enterRule(_localctx, 32, RULE_chain); try { int _alt; - setState(326); - switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { + setState(370); + switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { case 1: _localctx = new DynamicContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(292); + setState(336); ((DynamicContext)_localctx).p = primary(_localctx.c); - setState(296); + setState(340); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,19,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(293); + setState(337); secondary(((DynamicContext)_localctx).p.s); } } } - setState(298); + setState(342); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,19,_ctx); + _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } } break; @@ -1991,25 +2117,25 @@ class PainlessParser extends Parser { _localctx = new StaticContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(299); + setState(343); decltype(); - setState(300); + setState(344); dot(); - setState(304); + setState(348); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,24,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(301); + setState(345); secondary(true); } } } - setState(306); + setState(350); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,20,_ctx); + _alt = getInterpreter().adaptivePredict(_input,24,_ctx); } } break; @@ -2017,11 +2143,11 @@ class PainlessParser extends Parser { _localctx = new NewarrayContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(307); + setState(351); match(NEW); - setState(308); + setState(352); match(TYPE); - setState(313); + setState(357); _errHandler.sync(this); _alt = 1; do { @@ -2029,11 +2155,11 @@ class PainlessParser extends Parser { case 1: { { - setState(309); + setState(353); match(LBRACE); - setState(310); + setState(354); expression(0); - setState(311); + setState(355); match(RBRACE); } } @@ -2041,31 +2167,31 @@ class PainlessParser extends Parser { default: throw new NoViableAltException(this); } - setState(315); + setState(359); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,21,_ctx); + _alt = getInterpreter().adaptivePredict(_input,25,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); - setState(324); - switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { + setState(368); + switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { case 1: { - setState(317); + setState(361); dot(); - setState(321); + setState(365); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(318); + setState(362); secondary(true); } } } - setState(323); + setState(367); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,22,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); } } break; @@ -2102,6 +2228,15 @@ class PainlessParser extends Parser { this.s = ctx.s; } } + public static class RegexContext extends PrimaryContext { + public TerminalNode REGEX() { return getToken(PainlessParser.REGEX, 0); } + public RegexContext(PrimaryContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitRegex(this); + else return visitor.visitChildren(this); + } + } public static class StringContext extends PrimaryContext { public TerminalNode STRING() { return getToken(PainlessParser.STRING, 0); } public StringContext(PrimaryContext ctx) { copyFrom(ctx); } @@ -2111,6 +2246,18 @@ class PainlessParser extends Parser { else return visitor.visitChildren(this); } } + public static class CalllocalContext extends PrimaryContext { + public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } + public ArgumentsContext arguments() { + return getRuleContext(ArgumentsContext.class,0); + } + public CalllocalContext(PrimaryContext ctx) { copyFrom(ctx); } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitCalllocal(this); + else return visitor.visitChildren(this); + } + } public static class VariableContext extends PrimaryContext { public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } public VariableContext(PrimaryContext ctx) { copyFrom(ctx); } @@ -2163,21 +2310,21 @@ class PainlessParser extends Parser { public final PrimaryContext primary(boolean c) throws RecognitionException { PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c); - enterRule(_localctx, 32, RULE_primary); + enterRule(_localctx, 34, RULE_primary); try { - setState(344); - switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { + setState(391); + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: _localctx = new ExprprecContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(328); + setState(372); if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c "); - setState(329); + setState(373); match(LP); - setState(330); + setState(374); ((ExprprecContext)_localctx).e = expression(0); - setState(331); + setState(375); match(RP); ((ExprprecContext)_localctx).s = ((ExprprecContext)_localctx).e.s; } @@ -2186,13 +2333,13 @@ class PainlessParser extends Parser { _localctx = new ChainprecContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(334); + setState(378); if (!( _localctx.c )) throw new FailedPredicateException(this, " $c "); - setState(335); + setState(379); match(LP); - setState(336); + setState(380); unary(true); - setState(337); + setState(381); match(RP); } break; @@ -2200,27 +2347,45 @@ class PainlessParser extends Parser { _localctx = new StringContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(339); + setState(383); match(STRING); } break; case 4: - _localctx = new VariableContext(_localctx); + _localctx = new RegexContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(340); - match(ID); + setState(384); + match(REGEX); } break; case 5: - _localctx = new NewobjectContext(_localctx); + _localctx = new VariableContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(341); + setState(385); + match(ID); + } + break; + case 6: + _localctx = new CalllocalContext(_localctx); + enterOuterAlt(_localctx, 6); + { + setState(386); + match(ID); + setState(387); + arguments(); + } + break; + case 7: + _localctx = new NewobjectContext(_localctx); + enterOuterAlt(_localctx, 7); + { + setState(388); match(NEW); - setState(342); + setState(389); match(TYPE); - setState(343); + setState(390); arguments(); } break; @@ -2260,25 +2425,25 @@ class PainlessParser extends Parser { public final SecondaryContext secondary(boolean s) throws RecognitionException { SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s); - enterRule(_localctx, 34, RULE_secondary); + enterRule(_localctx, 36, RULE_secondary); try { - setState(350); - switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) { + setState(397); + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(346); + setState(393); if (!( _localctx.s )) throw new FailedPredicateException(this, " $s "); - setState(347); + setState(394); dot(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(348); + setState(395); if (!( _localctx.s )) throw new FailedPredicateException(this, " $s "); - setState(349); + setState(396); brace(); } break; @@ -2333,20 +2498,20 @@ class PainlessParser extends Parser { public final DotContext dot() throws RecognitionException { DotContext _localctx = new DotContext(_ctx, getState()); - enterRule(_localctx, 36, RULE_dot); + enterRule(_localctx, 38, RULE_dot); int _la; try { - setState(357); - switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) { + setState(404); + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: _localctx = new CallinvokeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(352); + setState(399); match(DOT); - setState(353); + setState(400); match(DOTID); - setState(354); + setState(401); arguments(); } break; @@ -2354,9 +2519,9 @@ class PainlessParser extends Parser { _localctx = new FieldaccessContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(355); + setState(402); match(DOT); - setState(356); + setState(403); _la = _input.LA(1); if ( !(_la==DOTINTEGER || _la==DOTID) ) { _errHandler.recoverInline(this); @@ -2405,16 +2570,16 @@ class PainlessParser extends Parser { public final BraceContext brace() throws RecognitionException { BraceContext _localctx = new BraceContext(_ctx, getState()); - enterRule(_localctx, 38, RULE_brace); + enterRule(_localctx, 40, RULE_brace); try { _localctx = new BraceaccessContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(359); + setState(406); match(LBRACE); - setState(360); + setState(407); expression(0); - setState(361); + setState(408); match(RBRACE); } } @@ -2455,40 +2620,40 @@ class PainlessParser extends Parser { public final ArgumentsContext arguments() throws RecognitionException { ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_arguments); + enterRule(_localctx, 42, RULE_arguments); int _la; try { enterOuterAlt(_localctx, 1); { { - setState(363); + setState(410); match(LP); - setState(372); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + setState(419); + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(364); + setState(411); argument(); - setState(369); + setState(416); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(365); + setState(412); match(COMMA); - setState(366); + setState(413); argument(); } } - setState(371); + setState(418); _errHandler.sync(this); _la = _input.LA(1); } } break; } - setState(374); + setState(421); match(RP); } } @@ -2508,6 +2673,9 @@ class PainlessParser extends Parser { public ExpressionContext expression() { return getRuleContext(ExpressionContext.class,0); } + public LambdaContext lambda() { + return getRuleContext(LambdaContext.class,0); + } public FuncrefContext funcref() { return getRuleContext(FuncrefContext.class,0); } @@ -2524,21 +2692,28 @@ class PainlessParser extends Parser { public final ArgumentContext argument() throws RecognitionException { ArgumentContext _localctx = new ArgumentContext(_ctx, getState()); - enterRule(_localctx, 42, RULE_argument); + enterRule(_localctx, 44, RULE_argument); try { - setState(378); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + setState(426); + switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(376); + setState(423); expression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(377); + setState(424); + lambda(); + } + break; + case 3: + enterOuterAlt(_localctx, 3); + { + setState(425); funcref(); } break; @@ -2555,17 +2730,396 @@ class PainlessParser extends Parser { return _localctx; } + public static class LambdaContext extends ParserRuleContext { + public TerminalNode ARROW() { return getToken(PainlessParser.ARROW, 0); } + public BlockContext block() { + return getRuleContext(BlockContext.class,0); + } + public List lamtype() { + return getRuleContexts(LamtypeContext.class); + } + public LamtypeContext lamtype(int i) { + return getRuleContext(LamtypeContext.class,i); + } + public TerminalNode LP() { return getToken(PainlessParser.LP, 0); } + public TerminalNode RP() { return getToken(PainlessParser.RP, 0); } + public List COMMA() { return getTokens(PainlessParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(PainlessParser.COMMA, i); + } + public LambdaContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_lambda; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitLambda(this); + else return visitor.visitChildren(this); + } + } + + public final LambdaContext lambda() throws RecognitionException { + LambdaContext _localctx = new LambdaContext(_ctx, getState()); + enterRule(_localctx, 46, RULE_lambda); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(441); + switch (_input.LA(1)) { + case TYPE: + case ID: + { + setState(428); + lamtype(); + } + break; + case LP: + { + setState(429); + match(LP); + setState(438); + _la = _input.LA(1); + if (_la==TYPE || _la==ID) { + { + setState(430); + lamtype(); + setState(435); + _errHandler.sync(this); + _la = _input.LA(1); + while (_la==COMMA) { + { + { + setState(431); + match(COMMA); + setState(432); + lamtype(); + } + } + setState(437); + _errHandler.sync(this); + _la = _input.LA(1); + } + } + } + + setState(440); + match(RP); + } + break; + default: + throw new NoViableAltException(this); + } + setState(443); + match(ARROW); + setState(444); + block(); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class LamtypeContext extends ParserRuleContext { + public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public LamtypeContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_lamtype; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitLamtype(this); + else return visitor.visitChildren(this); + } + } + + public final LamtypeContext lamtype() throws RecognitionException { + LamtypeContext _localctx = new LamtypeContext(_ctx, getState()); + enterRule(_localctx, 48, RULE_lamtype); + int _la; + try { + enterOuterAlt(_localctx, 1); + { + setState(447); + _la = _input.LA(1); + if (_la==TYPE) { + { + setState(446); + decltype(); + } + } + + setState(449); + match(ID); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class FuncrefContext extends ParserRuleContext { + public ClassFuncrefContext classFuncref() { + return getRuleContext(ClassFuncrefContext.class,0); + } + public ConstructorFuncrefContext constructorFuncref() { + return getRuleContext(ConstructorFuncrefContext.class,0); + } + public CapturingFuncrefContext capturingFuncref() { + return getRuleContext(CapturingFuncrefContext.class,0); + } + public LocalFuncrefContext localFuncref() { + return getRuleContext(LocalFuncrefContext.class,0); + } + public FuncrefContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_funcref; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitFuncref(this); + else return visitor.visitChildren(this); + } + } + + public final FuncrefContext funcref() throws RecognitionException { + FuncrefContext _localctx = new FuncrefContext(_ctx, getState()); + enterRule(_localctx, 50, RULE_funcref); + try { + setState(455); + switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) { + case 1: + enterOuterAlt(_localctx, 1); + { + setState(451); + classFuncref(); + } + break; + case 2: + enterOuterAlt(_localctx, 2); + { + setState(452); + constructorFuncref(); + } + break; + case 3: + enterOuterAlt(_localctx, 3); + { + setState(453); + capturingFuncref(); + } + break; + case 4: + enterOuterAlt(_localctx, 4); + { + setState(454); + localFuncref(); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ClassFuncrefContext extends ParserRuleContext { + public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); } + public TerminalNode REF() { return getToken(PainlessParser.REF, 0); } + public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } + public ClassFuncrefContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_classFuncref; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitClassFuncref(this); + else return visitor.visitChildren(this); + } + } + + public final ClassFuncrefContext classFuncref() throws RecognitionException { + ClassFuncrefContext _localctx = new ClassFuncrefContext(_ctx, getState()); + enterRule(_localctx, 52, RULE_classFuncref); + try { + enterOuterAlt(_localctx, 1); + { + setState(457); + match(TYPE); + setState(458); + match(REF); + setState(459); + match(ID); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ConstructorFuncrefContext extends ParserRuleContext { + public DecltypeContext decltype() { + return getRuleContext(DecltypeContext.class,0); + } + public TerminalNode REF() { return getToken(PainlessParser.REF, 0); } + public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); } + public ConstructorFuncrefContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_constructorFuncref; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitConstructorFuncref(this); + else return visitor.visitChildren(this); + } + } + + public final ConstructorFuncrefContext constructorFuncref() throws RecognitionException { + ConstructorFuncrefContext _localctx = new ConstructorFuncrefContext(_ctx, getState()); + enterRule(_localctx, 54, RULE_constructorFuncref); + try { + enterOuterAlt(_localctx, 1); + { + setState(461); + decltype(); + setState(462); + match(REF); + setState(463); + match(NEW); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class CapturingFuncrefContext extends ParserRuleContext { + public List ID() { return getTokens(PainlessParser.ID); } + public TerminalNode ID(int i) { + return getToken(PainlessParser.ID, i); + } + public TerminalNode REF() { return getToken(PainlessParser.REF, 0); } + public CapturingFuncrefContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_capturingFuncref; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitCapturingFuncref(this); + else return visitor.visitChildren(this); + } + } + + public final CapturingFuncrefContext capturingFuncref() throws RecognitionException { + CapturingFuncrefContext _localctx = new CapturingFuncrefContext(_ctx, getState()); + enterRule(_localctx, 56, RULE_capturingFuncref); + try { + enterOuterAlt(_localctx, 1); + { + setState(465); + match(ID); + setState(466); + match(REF); + setState(467); + match(ID); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class LocalFuncrefContext extends ParserRuleContext { + public TerminalNode THIS() { return getToken(PainlessParser.THIS, 0); } + public TerminalNode REF() { return getToken(PainlessParser.REF, 0); } + public TerminalNode ID() { return getToken(PainlessParser.ID, 0); } + public LocalFuncrefContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_localFuncref; } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor)visitor).visitLocalFuncref(this); + else return visitor.visitChildren(this); + } + } + + public final LocalFuncrefContext localFuncref() throws RecognitionException { + LocalFuncrefContext _localctx = new LocalFuncrefContext(_ctx, getState()); + enterRule(_localctx, 58, RULE_localFuncref); + try { + enterOuterAlt(_localctx, 1); + { + setState(469); + match(THIS); + setState(470); + match(REF); + setState(471); + match(ID); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 1: + case 3: return statement_sempred((StatementContext)_localctx, predIndex); - case 13: - return expression_sempred((ExpressionContext)_localctx, predIndex); case 14: + return expression_sempred((ExpressionContext)_localctx, predIndex); + case 15: return unary_sempred((UnaryContext)_localctx, predIndex); - case 16: - return primary_sempred((PrimaryContext)_localctx, predIndex); case 17: + return primary_sempred((PrimaryContext)_localctx, predIndex); + case 18: return secondary_sempred((SecondaryContext)_localctx, predIndex); } return true; @@ -2645,146 +3199,184 @@ class PainlessParser extends Parser { } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u017f\4\2\t\2\4"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3P\u01dc\4\2\t\2\4"+ "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+ "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ - "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\3\2\7\2\60\n\2\f\2"+ - "\16\2\63\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3?\n\3\3\3\3\3"+ - "\3\3\3\3\3\3\3\3\5\3G\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+ - "\5\3T\n\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\3\3\5\3a\n\3\3\3"+ - "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+ - "\3\3\3\3\3\3\3\3\3\6\3z\n\3\r\3\16\3{\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3"+ - "\u0085\n\3\3\4\3\4\5\4\u0089\n\4\3\5\3\5\7\5\u008d\n\5\f\5\16\5\u0090"+ - "\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u0098\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+ - "\t\u00a0\n\t\f\t\16\t\u00a3\13\t\3\n\3\n\3\n\7\n\u00a8\n\n\f\n\16\n\u00ab"+ - "\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f\u00b4\n\f\3\r\3\r\3\r\3\r\3"+ - "\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17"+ - "\u00c8\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+ - "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+ - "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+ - "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+ - "\3\17\3\17\3\17\7\17\u0103\n\17\f\17\16\17\u0106\13\17\3\20\3\20\3\20"+ + "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+ + "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\3\2\7\2@"+ + "\n\2\f\2\16\2C\13\2\3\2\7\2F\n\2\f\2\16\2I\13\2\3\2\3\2\3\3\3\3\3\3\3"+ + "\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4Y\n\4\f\4\16\4\\\13\4\5\4^\n\4\3"+ + "\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5j\n\5\3\5\3\5\3\5\3\5\3\5\3"+ + "\5\5\5r\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5\177\n\5\3"+ + "\5\3\5\5\5\u0083\n\5\3\5\3\5\5\5\u0087\n\5\3\5\3\5\3\5\5\5\u008c\n\5\3"+ + "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+ + "\3\5\3\5\3\5\3\5\3\5\6\5\u00a5\n\5\r\5\16\5\u00a6\3\5\3\5\3\5\3\5\3\5"+ + "\3\5\3\5\5\5\u00b0\n\5\3\6\3\6\5\6\u00b4\n\6\3\7\3\7\7\7\u00b8\n\7\f\7"+ + "\16\7\u00bb\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00c3\n\t\3\n\3\n\3\13\3"+ + "\13\3\13\3\13\7\13\u00cb\n\13\f\13\16\13\u00ce\13\13\3\f\3\f\3\f\7\f\u00d3"+ + "\n\f\f\f\16\f\u00d6\13\f\3\r\3\r\3\r\5\r\u00db\n\r\3\16\3\16\3\16\3\16"+ + "\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\5\17\u00e9\n\17\3\20\3\20\3\20"+ + "\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00f4\n\20\3\20\3\20\3\20\3\20\3\20"+ "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+ - "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u0125"+ - "\n\20\3\21\3\21\7\21\u0129\n\21\f\21\16\21\u012c\13\21\3\21\3\21\3\21"+ - "\7\21\u0131\n\21\f\21\16\21\u0134\13\21\3\21\3\21\3\21\3\21\3\21\3\21"+ - "\6\21\u013c\n\21\r\21\16\21\u013d\3\21\3\21\7\21\u0142\n\21\f\21\16\21"+ - "\u0145\13\21\5\21\u0147\n\21\5\21\u0149\n\21\3\22\3\22\3\22\3\22\3\22"+ - "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u015b\n\22"+ - "\3\23\3\23\3\23\3\23\5\23\u0161\n\23\3\24\3\24\3\24\3\24\3\24\5\24\u0168"+ - "\n\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\7\26\u0172\n\26\f\26\16"+ - "\26\u0175\13\26\5\26\u0177\n\26\3\26\3\26\3\27\3\27\5\27\u017d\n\27\3"+ - "\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,\2\16\4\2"+ - "\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\64\65"+ - "\3\2BE\4\2\32\33\37 \3\2LM\u01a5\2\61\3\2\2\2\4\u0084\3\2\2\2\6\u0088"+ - "\3\2\2\2\b\u008a\3\2\2\2\n\u0093\3\2\2\2\f\u0097\3\2\2\2\16\u0099\3\2"+ - "\2\2\20\u009b\3\2\2\2\22\u00a4\3\2\2\2\24\u00ac\3\2\2\2\26\u00b0\3\2\2"+ - "\2\30\u00b5\3\2\2\2\32\u00bc\3\2\2\2\34\u00c7\3\2\2\2\36\u0124\3\2\2\2"+ - " \u0148\3\2\2\2\"\u015a\3\2\2\2$\u0160\3\2\2\2&\u0167\3\2\2\2(\u0169\3"+ - "\2\2\2*\u016d\3\2\2\2,\u017c\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60\63\3\2\2"+ - "\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65\7\2\2\3"+ - "\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2:>\5\6\4"+ - "\2;<\7\17\2\2;\3\2\2\2>=\3\2\2\2?\u0085\3\2\2\2@"+ - "A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3\2\2"+ - "\2FE\3\2\2\2G\u0085\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2"+ - "LM\5\34\17\2MN\7\n\2\2NO\5\32\16\2O\u0085\3\2\2\2PQ\7\22\2\2QS\7\t\2\2"+ - "RT\5\f\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2"+ - "\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3"+ - "\2\2\2]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a\u0085\3\2\2"+ - "\2bc\7\22\2\2cd\7\t\2\2de\5\22\n\2ef\7K\2\2fg\7\62\2\2gh\5\34\17\2hi\7"+ - "\n\2\2ij\5\6\4\2j\u0085\3\2\2\2kl\5\20\t\2lm\5\32\16\2m\u0085\3\2\2\2"+ - "no\7\23\2\2o\u0085\5\32\16\2pq\7\24\2\2q\u0085\5\32\16\2rs\7\25\2\2st"+ - "\5\34\17\2tu\5\32\16\2u\u0085\3\2\2\2vw\7\27\2\2wy\5\b\5\2xz\5\30\r\2"+ - "yx\3\2\2\2z{\3\2\2\2{y\3\2\2\2{|\3\2\2\2|\u0085\3\2\2\2}~\7\31\2\2~\177"+ - "\5\34\17\2\177\u0080\5\32\16\2\u0080\u0085\3\2\2\2\u0081\u0082\5\34\17"+ - "\2\u0082\u0083\5\32\16\2\u0083\u0085\3\2\2\2\u0084\66\3\2\2\2\u0084@\3"+ - "\2\2\2\u0084H\3\2\2\2\u0084P\3\2\2\2\u0084b\3\2\2\2\u0084k\3\2\2\2\u0084"+ - "n\3\2\2\2\u0084p\3\2\2\2\u0084r\3\2\2\2\u0084v\3\2\2\2\u0084}\3\2\2\2"+ - "\u0084\u0081\3\2\2\2\u0085\5\3\2\2\2\u0086\u0089\5\b\5\2\u0087\u0089\5"+ - "\4\3\2\u0088\u0086\3\2\2\2\u0088\u0087\3\2\2\2\u0089\7\3\2\2\2\u008a\u008e"+ - "\7\5\2\2\u008b\u008d\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u0090\3\2\2\2\u008e"+ - "\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2\2\2\u0090\u008e\3\2"+ - "\2\2\u0091\u0092\7\6\2\2\u0092\t\3\2\2\2\u0093\u0094\7\r\2\2\u0094\13"+ - "\3\2\2\2\u0095\u0098\5\20\t\2\u0096\u0098\5\34\17\2\u0097\u0095\3\2\2"+ - "\2\u0097\u0096\3\2\2\2\u0098\r\3\2\2\2\u0099\u009a\5\34\17\2\u009a\17"+ - "\3\2\2\2\u009b\u009c\5\22\n\2\u009c\u00a1\5\26\f\2\u009d\u009e\7\f\2\2"+ - "\u009e\u00a0\5\26\f\2\u009f\u009d\3\2\2\2\u00a0\u00a3\3\2\2\2\u00a1\u009f"+ - "\3\2\2\2\u00a1\u00a2\3\2\2\2\u00a2\21\3\2\2\2\u00a3\u00a1\3\2\2\2\u00a4"+ - "\u00a9\7J\2\2\u00a5\u00a6\7\7\2\2\u00a6\u00a8\7\b\2\2\u00a7\u00a5\3\2"+ - "\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa"+ - "\23\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00ad\7J\2\2\u00ad\u00ae\7\63\2"+ - "\2\u00ae\u00af\t\2\2\2\u00af\25\3\2\2\2\u00b0\u00b3\7K\2\2\u00b1\u00b2"+ - "\7\66\2\2\u00b2\u00b4\5\34\17\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2"+ - "\2\u00b4\27\3\2\2\2\u00b5\u00b6\7\30\2\2\u00b6\u00b7\7\t\2\2\u00b7\u00b8"+ - "\7J\2\2\u00b8\u00b9\7K\2\2\u00b9\u00ba\7\n\2\2\u00ba\u00bb\5\b\5\2\u00bb"+ - "\31\3\2\2\2\u00bc\u00bd\t\3\2\2\u00bd\33\3\2\2\2\u00be\u00bf\b\17\1\2"+ - "\u00bf\u00c0\5 \21\2\u00c0\u00c1\t\4\2\2\u00c1\u00c2\5\34\17\3\u00c2\u00c3"+ - "\b\17\1\2\u00c3\u00c8\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\u00c6\b\17\1"+ - "\2\u00c6\u00c8\3\2\2\2\u00c7\u00be\3\2\2\2\u00c7\u00c4\3\2\2\2\u00c8\u0104"+ - "\3\2\2\2\u00c9\u00ca\f\16\2\2\u00ca\u00cb\t\5\2\2\u00cb\u00cc\5\34\17"+ - "\17\u00cc\u00cd\b\17\1\2\u00cd\u0103\3\2\2\2\u00ce\u00cf\f\r\2\2\u00cf"+ - "\u00d0\t\6\2\2\u00d0\u00d1\5\34\17\16\u00d1\u00d2\b\17\1\2\u00d2\u0103"+ - "\3\2\2\2\u00d3\u00d4\f\f\2\2\u00d4\u00d5\t\7\2\2\u00d5\u00d6\5\34\17\r"+ - "\u00d6\u00d7\b\17\1\2\u00d7\u0103\3\2\2\2\u00d8\u00d9\f\13\2\2\u00d9\u00da"+ - "\t\b\2\2\u00da\u00db\5\34\17\f\u00db\u00dc\b\17\1\2\u00dc\u0103\3\2\2"+ - "\2\u00dd\u00de\f\n\2\2\u00de\u00df\t\t\2\2\u00df\u00e0\5\34\17\13\u00e0"+ - "\u00e1\b\17\1\2\u00e1\u0103\3\2\2\2\u00e2\u00e3\f\t\2\2\u00e3\u00e4\7"+ - ",\2\2\u00e4\u00e5\5\34\17\n\u00e5\u00e6\b\17\1\2\u00e6\u0103\3\2\2\2\u00e7"+ - "\u00e8\f\b\2\2\u00e8\u00e9\7-\2\2\u00e9\u00ea\5\34\17\t\u00ea\u00eb\b"+ - "\17\1\2\u00eb\u0103\3\2\2\2\u00ec\u00ed\f\7\2\2\u00ed\u00ee\7.\2\2\u00ee"+ - "\u00ef\5\34\17\b\u00ef\u00f0\b\17\1\2\u00f0\u0103\3\2\2\2\u00f1\u00f2"+ - "\f\6\2\2\u00f2\u00f3\7/\2\2\u00f3\u00f4\5\34\17\7\u00f4\u00f5\b\17\1\2"+ - "\u00f5\u0103\3\2\2\2\u00f6\u00f7\f\5\2\2\u00f7\u00f8\7\60\2\2\u00f8\u00f9"+ - "\5\34\17\6\u00f9\u00fa\b\17\1\2\u00fa\u0103\3\2\2\2\u00fb\u00fc\f\4\2"+ - "\2\u00fc\u00fd\7\61\2\2\u00fd\u00fe\5\34\17\2\u00fe\u00ff\7\62\2\2\u00ff"+ - "\u0100\5\34\17\4\u0100\u0101\b\17\1\2\u0101\u0103\3\2\2\2\u0102\u00c9"+ - "\3\2\2\2\u0102\u00ce\3\2\2\2\u0102\u00d3\3\2\2\2\u0102\u00d8\3\2\2\2\u0102"+ - "\u00dd\3\2\2\2\u0102\u00e2\3\2\2\2\u0102\u00e7\3\2\2\2\u0102\u00ec\3\2"+ - "\2\2\u0102\u00f1\3\2\2\2\u0102\u00f6\3\2\2\2\u0102\u00fb\3\2\2\2\u0103"+ - "\u0106\3\2\2\2\u0104\u0102\3\2\2\2\u0104\u0105\3\2\2\2\u0105\35\3\2\2"+ - "\2\u0106\u0104\3\2\2\2\u0107\u0108\6\20\16\3\u0108\u0109\t\n\2\2\u0109"+ - "\u0125\5 \21\2\u010a\u010b\6\20\17\3\u010b\u010c\5 \21\2\u010c\u010d\t"+ - "\n\2\2\u010d\u0125\3\2\2\2\u010e\u010f\6\20\20\3\u010f\u0125\5 \21\2\u0110"+ - "\u0111\6\20\21\3\u0111\u0112\t\13\2\2\u0112\u0125\b\20\1\2\u0113\u0114"+ - "\6\20\22\3\u0114\u0115\7G\2\2\u0115\u0125\b\20\1\2\u0116\u0117\6\20\23"+ - "\3\u0117\u0118\7H\2\2\u0118\u0125\b\20\1\2\u0119\u011a\6\20\24\3\u011a"+ - "\u011b\7I\2\2\u011b\u0125\b\20\1\2\u011c\u011d\6\20\25\3\u011d\u011e\t"+ - "\f\2\2\u011e\u0125\5\36\20\2\u011f\u0120\7\t\2\2\u0120\u0121\5\22\n\2"+ - "\u0121\u0122\7\n\2\2\u0122\u0123\5\36\20\2\u0123\u0125\3\2\2\2\u0124\u0107"+ - "\3\2\2\2\u0124\u010a\3\2\2\2\u0124\u010e\3\2\2\2\u0124\u0110\3\2\2\2\u0124"+ - "\u0113\3\2\2\2\u0124\u0116\3\2\2\2\u0124\u0119\3\2\2\2\u0124\u011c\3\2"+ - "\2\2\u0124\u011f\3\2\2\2\u0125\37\3\2\2\2\u0126\u012a\5\"\22\2\u0127\u0129"+ - "\5$\23\2\u0128\u0127\3\2\2\2\u0129\u012c\3\2\2\2\u012a\u0128\3\2\2\2\u012a"+ - "\u012b\3\2\2\2\u012b\u0149\3\2\2\2\u012c\u012a\3\2\2\2\u012d\u012e\5\22"+ - "\n\2\u012e\u0132\5&\24\2\u012f\u0131\5$\23\2\u0130\u012f\3\2\2\2\u0131"+ - "\u0134\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0149\3\2"+ - "\2\2\u0134\u0132\3\2\2\2\u0135\u0136\7\26\2\2\u0136\u013b\7J\2\2\u0137"+ - "\u0138\7\7\2\2\u0138\u0139\5\34\17\2\u0139\u013a\7\b\2\2\u013a\u013c\3"+ - "\2\2\2\u013b\u0137\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u013b\3\2\2\2\u013d"+ - "\u013e\3\2\2\2\u013e\u0146\3\2\2\2\u013f\u0143\5&\24\2\u0140\u0142\5$"+ - "\23\2\u0141\u0140\3\2\2\2\u0142\u0145\3\2\2\2\u0143\u0141\3\2\2\2\u0143"+ - "\u0144\3\2\2\2\u0144\u0147\3\2\2\2\u0145\u0143\3\2\2\2\u0146\u013f\3\2"+ - "\2\2\u0146\u0147\3\2\2\2\u0147\u0149\3\2\2\2\u0148\u0126\3\2\2\2\u0148"+ - "\u012d\3\2\2\2\u0148\u0135\3\2\2\2\u0149!\3\2\2\2\u014a\u014b\6\22\26"+ - "\3\u014b\u014c\7\t\2\2\u014c\u014d\5\34\17\2\u014d\u014e\7\n\2\2\u014e"+ - "\u014f\b\22\1\2\u014f\u015b\3\2\2\2\u0150\u0151\6\22\27\3\u0151\u0152"+ - "\7\t\2\2\u0152\u0153\5\36\20\2\u0153\u0154\7\n\2\2\u0154\u015b\3\2\2\2"+ - "\u0155\u015b\7F\2\2\u0156\u015b\7K\2\2\u0157\u0158\7\26\2\2\u0158\u0159"+ - "\7J\2\2\u0159\u015b\5*\26\2\u015a\u014a\3\2\2\2\u015a\u0150\3\2\2\2\u015a"+ - "\u0155\3\2\2\2\u015a\u0156\3\2\2\2\u015a\u0157\3\2\2\2\u015b#\3\2\2\2"+ - "\u015c\u015d\6\23\30\3\u015d\u0161\5&\24\2\u015e\u015f\6\23\31\3\u015f"+ - "\u0161\5(\25\2\u0160\u015c\3\2\2\2\u0160\u015e\3\2\2\2\u0161%\3\2\2\2"+ - "\u0162\u0163\7\13\2\2\u0163\u0164\7M\2\2\u0164\u0168\5*\26\2\u0165\u0166"+ - "\7\13\2\2\u0166\u0168\t\r\2\2\u0167\u0162\3\2\2\2\u0167\u0165\3\2\2\2"+ - "\u0168\'\3\2\2\2\u0169\u016a\7\7\2\2\u016a\u016b\5\34\17\2\u016b\u016c"+ - "\7\b\2\2\u016c)\3\2\2\2\u016d\u0176\7\t\2\2\u016e\u0173\5,\27\2\u016f"+ - "\u0170\7\f\2\2\u0170\u0172\5,\27\2\u0171\u016f\3\2\2\2\u0172\u0175\3\2"+ - "\2\2\u0173\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0177\3\2\2\2\u0175"+ - "\u0173\3\2\2\2\u0176\u016e\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0178\3\2"+ - "\2\2\u0178\u0179\7\n\2\2\u0179+\3\2\2\2\u017a\u017d\5\34\17\2\u017b\u017d"+ - "\5\24\13\2\u017c\u017a\3\2\2\2\u017c\u017b\3\2\2\2\u017d-\3\2\2\2!\61"+ - ">FSW[`{\u0084\u0088\u008e\u0097\u00a1\u00a9\u00b3\u00c7\u0102\u0104\u0124"+ - "\u012a\u0132\u013d\u0143\u0146\u0148\u015a\u0160\u0167\u0173\u0176\u017c"; + "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+ + "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+ + "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u012f\n\20\f\20"+ + "\16\20\u0132\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+ + "\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+ + "\21\3\21\3\21\3\21\3\21\5\21\u0151\n\21\3\22\3\22\7\22\u0155\n\22\f\22"+ + "\16\22\u0158\13\22\3\22\3\22\3\22\7\22\u015d\n\22\f\22\16\22\u0160\13"+ + "\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22"+ + "\3\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+ + "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+ + "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018a\n\23\3\24\3\24\3\24\3\24\5\24"+ + "\u0190\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0197\n\25\3\26\3\26\3\26\3"+ + "\26\3\27\3\27\3\27\3\27\7\27\u01a1\n\27\f\27\16\27\u01a4\13\27\5\27\u01a6"+ + "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ad\n\30\3\31\3\31\3\31\3\31\3\31"+ + "\7\31\u01b4\n\31\f\31\16\31\u01b7\13\31\5\31\u01b9\n\31\3\31\5\31\u01bc"+ + "\n\31\3\31\3\31\3\31\3\32\5\32\u01c2\n\32\3\32\3\32\3\33\3\33\3\33\3\33"+ + "\5\33\u01ca\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36"+ + "\3\36\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32"+ + "\34\36 \"$&(*,.\60\62\64\668:<\2\f\3\28C\3\2\35\37\3\2 !\3\2\"$\3\2%("+ + "\3\2),\3\2\66\67\3\2DG\4\2\33\34 !\3\2OP\u0209\2A\3\2\2\2\4L\3\2\2\2\6"+ + "Q\3\2\2\2\b\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2"+ + "\2\2\20\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2"+ + "\2\30\u00d7\3\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2"+ + " \u0150\3\2\2\2\"\u0174\3\2\2\2$\u0189\3\2\2\2&\u018f\3\2\2\2(\u0196\3"+ + "\2\2\2*\u0198\3\2\2\2,\u019c\3\2\2\2.\u01ac\3\2\2\2\60\u01bb\3\2\2\2\62"+ + "\u01c1\3\2\2\2\64\u01c9\3\2\2\2\66\u01cb\3\2\2\28\u01cf\3\2\2\2:\u01d3"+ + "\3\2\2\2<\u01d7\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2"+ + "\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2"+ + "\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7N\2\2NO\5"+ + "\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7N\2\2TU\7\f\2\2UV"+ + "\5\26\f\2VW\7N\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2"+ + "[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2"+ + "\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5"+ + "\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2"+ + "\2mn\5\36\20\2nq\7\n\2\2or\5\n\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0"+ + "\3\2\2\2st\7\21\2\2tu\5\f\7\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2"+ + "\2yz\5\34\17\2z\u00b0\3\2\2\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2"+ + "\2\2~\177\3\2\2\2\177\u0080\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5"+ + "\36\20\2\u0082\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084"+ + "\u0086\7\r\2\2\u0085\u0087\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3"+ + "\2\2\2\u0087\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a"+ + "\u008c\5\16\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3"+ + "\2\2\2\u008d\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090"+ + "\u0091\7N\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7"+ + "\n\2\2\u0094\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097"+ + "\u0098\5\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0"+ + "\5\34\17\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25"+ + "\2\2\u009e\u009f\5\36\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1"+ + "\u00a2\7\27\2\2\u00a2\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3"+ + "\3\2\2\2\u00a5\u00a6\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7"+ + "\u00b0\3\2\2\2\u00a8\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab"+ + "\5\34\17\2\u00ab\u00b0\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34"+ + "\17\2\u00ae\u00b0\3\2\2\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2"+ + "\u00af{\3\2\2\2\u00af\u008d\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3"+ + "\2\2\2\u00af\u009b\3\2\2\2\u00af\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af"+ + "\u00a8\3\2\2\2\u00af\u00ac\3\2\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2"+ + "\u00b2\u00b4\5\b\5\2\u00b3\u00b1\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13"+ + "\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8"+ + "\u00bb\3\2\2\2\u00b9\u00b7\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2"+ + "\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf"+ + "\7\r\2\2\u00bf\17\3\2\2\2\u00c0\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2"+ + "\u00c2\u00c0\3\2\2\2\u00c2\u00c1\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5"+ + "\5\36\20\2\u00c5\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2"+ + "\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+ + "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce"+ + "\u00cc\3\2\2\2\u00cf\u00d4\7M\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b"+ + "\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+ + "\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7N\2\2"+ + "\u00d8\u00d9\78\2\2\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db"+ + "\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de"+ + "\u00df\7M\2\2\u00df\u00e0\7N\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7"+ + "\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6"+ + "\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2"+ + "\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb"+ + "\b\20\1\2\u00eb\u00ec\5\"\22\2\u00ec\u00ed\t\2\2\2\u00ed\u00ee\5\36\20"+ + "\3\u00ee\u00ef\b\20\1\2\u00ef\u00f4\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1"+ + "\u00f2\b\20\1\2\u00f2\u00f4\3\2\2\2\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3"+ + "\2\2\2\u00f4\u0130\3\2\2\2\u00f5\u00f6\f\16\2\2\u00f6\u00f7\t\3\2\2\u00f7"+ + "\u00f8\5\36\20\17\u00f8\u00f9\b\20\1\2\u00f9\u012f\3\2\2\2\u00fa\u00fb"+ + "\f\r\2\2\u00fb\u00fc\t\4\2\2\u00fc\u00fd\5\36\20\16\u00fd\u00fe\b\20\1"+ + "\2\u00fe\u012f\3\2\2\2\u00ff\u0100\f\f\2\2\u0100\u0101\t\5\2\2\u0101\u0102"+ + "\5\36\20\r\u0102\u0103\b\20\1\2\u0103\u012f\3\2\2\2\u0104\u0105\f\13\2"+ + "\2\u0105\u0106\t\6\2\2\u0106\u0107\5\36\20\f\u0107\u0108\b\20\1\2\u0108"+ + "\u012f\3\2\2\2\u0109\u010a\f\n\2\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36"+ + "\20\13\u010c\u010d\b\20\1\2\u010d\u012f\3\2\2\2\u010e\u010f\f\t\2\2\u010f"+ + "\u0110\7-\2\2\u0110\u0111\5\36\20\n\u0111\u0112\b\20\1\2\u0112\u012f\3"+ + "\2\2\2\u0113\u0114\f\b\2\2\u0114\u0115\7.\2\2\u0115\u0116\5\36\20\t\u0116"+ + "\u0117\b\20\1\2\u0117\u012f\3\2\2\2\u0118\u0119\f\7\2\2\u0119\u011a\7"+ + "/\2\2\u011a\u011b\5\36\20\b\u011b\u011c\b\20\1\2\u011c\u012f\3\2\2\2\u011d"+ + "\u011e\f\6\2\2\u011e\u011f\7\60\2\2\u011f\u0120\5\36\20\7\u0120\u0121"+ + "\b\20\1\2\u0121\u012f\3\2\2\2\u0122\u0123\f\5\2\2\u0123\u0124\7\61\2\2"+ + "\u0124\u0125\5\36\20\6\u0125\u0126\b\20\1\2\u0126\u012f\3\2\2\2\u0127"+ + "\u0128\f\4\2\2\u0128\u0129\7\62\2\2\u0129\u012a\5\36\20\2\u012a\u012b"+ + "\7\63\2\2\u012b\u012c\5\36\20\4\u012c\u012d\b\20\1\2\u012d\u012f\3\2\2"+ + "\2\u012e\u00f5\3\2\2\2\u012e\u00fa\3\2\2\2\u012e\u00ff\3\2\2\2\u012e\u0104"+ + "\3\2\2\2\u012e\u0109\3\2\2\2\u012e\u010e\3\2\2\2\u012e\u0113\3\2\2\2\u012e"+ + "\u0118\3\2\2\2\u012e\u011d\3\2\2\2\u012e\u0122\3\2\2\2\u012e\u0127\3\2"+ + "\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130\u0131\3\2\2\2\u0131"+ + "\37\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0134\6\21\16\3\u0134\u0135\t\b"+ + "\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\17\3\u0137\u0138\5\"\22\2\u0138"+ + "\u0139\t\b\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6\21\20\3\u013b\u0151\5"+ + "\"\22\2\u013c\u013d\6\21\21\3\u013d\u013e\t\t\2\2\u013e\u0151\b\21\1\2"+ + "\u013f\u0140\6\21\22\3\u0140\u0141\7J\2\2\u0141\u0151\b\21\1\2\u0142\u0143"+ + "\6\21\23\3\u0143\u0144\7K\2\2\u0144\u0151\b\21\1\2\u0145\u0146\6\21\24"+ + "\3\u0146\u0147\7L\2\2\u0147\u0151\b\21\1\2\u0148\u0149\6\21\25\3\u0149"+ + "\u014a\t\n\2\2\u014a\u0151\5 \21\2\u014b\u014c\7\t\2\2\u014c\u014d\5\26"+ + "\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2\u014f\u0151\3\2\2\2\u0150"+ + "\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a\3\2\2\2\u0150\u013c\3\2"+ + "\2\2\u0150\u013f\3\2\2\2\u0150\u0142\3\2\2\2\u0150\u0145\3\2\2\2\u0150"+ + "\u0148\3\2\2\2\u0150\u014b\3\2\2\2\u0151!\3\2\2\2\u0152\u0156\5$\23\2"+ + "\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158\3\2\2\2\u0156\u0154"+ + "\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0175\3\2\2\2\u0158\u0156\3\2\2\2\u0159"+ + "\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5&\24\2\u015c\u015b\3"+ + "\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+ + "\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26\2\2\u0162\u0167\7"+ + "M\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165\u0166\7\b\2\2\u0166"+ + "\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u0167\3\2"+ + "\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b\u016f\5(\25\2\u016c"+ + "\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2\2\2\u016f\u016d\3\2"+ + "\2\2\u016f\u0170\3\2\2\2\u0170\u0173\3\2\2\2\u0171\u016f\3\2\2\2\u0172"+ + "\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2\2\2\u0174\u0152\3\2"+ + "\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175#\3\2\2\2\u0176\u0177"+ + "\6\23\26\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20\2\u0179\u017a\7\n\2"+ + "\2\u017a\u017b\b\23\1\2\u017b\u018a\3\2\2\2\u017c\u017d\6\23\27\3\u017d"+ + "\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7\n\2\2\u0180\u018a\3\2"+ + "\2\2\u0181\u018a\7H\2\2\u0182\u018a\7I\2\2\u0183\u018a\7N\2\2\u0184\u0185"+ + "\7N\2\2\u0185\u018a\5,\27\2\u0186\u0187\7\26\2\2\u0187\u0188\7M\2\2\u0188"+ + "\u018a\5,\27\2\u0189\u0176\3\2\2\2\u0189\u017c\3\2\2\2\u0189\u0181\3\2"+ + "\2\2\u0189\u0182\3\2\2\2\u0189\u0183\3\2\2\2\u0189\u0184\3\2\2\2\u0189"+ + "\u0186\3\2\2\2\u018a%\3\2\2\2\u018b\u018c\6\24\30\3\u018c\u0190\5(\25"+ + "\2\u018d\u018e\6\24\31\3\u018e\u0190\5*\26\2\u018f\u018b\3\2\2\2\u018f"+ + "\u018d\3\2\2\2\u0190\'\3\2\2\2\u0191\u0192\7\13\2\2\u0192\u0193\7P\2\2"+ + "\u0193\u0197\5,\27\2\u0194\u0195\7\13\2\2\u0195\u0197\t\13\2\2\u0196\u0191"+ + "\3\2\2\2\u0196\u0194\3\2\2\2\u0197)\3\2\2\2\u0198\u0199\7\7\2\2\u0199"+ + "\u019a\5\36\20\2\u019a\u019b\7\b\2\2\u019b+\3\2\2\2\u019c\u01a5\7\t\2"+ + "\2\u019d\u01a2\5.\30\2\u019e\u019f\7\f\2\2\u019f\u01a1\5.\30\2\u01a0\u019e"+ + "\3\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+ + "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u01a6\3\2"+ + "\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a8\7\n\2\2\u01a8-\3\2\2\2\u01a9\u01ad"+ + "\5\36\20\2\u01aa\u01ad\5\60\31\2\u01ab\u01ad\5\64\33\2\u01ac\u01a9\3\2"+ + "\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ab\3\2\2\2\u01ad/\3\2\2\2\u01ae\u01bc"+ + "\5\62\32\2\u01af\u01b8\7\t\2\2\u01b0\u01b5\5\62\32\2\u01b1\u01b2\7\f\2"+ + "\2\u01b2\u01b4\5\62\32\2\u01b3\u01b1\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5"+ + "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b9\3\2\2\2\u01b7\u01b5\3\2"+ + "\2\2\u01b8\u01b0\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba"+ + "\u01bc\7\n\2\2\u01bb\u01ae\3\2\2\2\u01bb\u01af\3\2\2\2\u01bc\u01bd\3\2"+ + "\2\2\u01bd\u01be\7\65\2\2\u01be\u01bf\5\f\7\2\u01bf\61\3\2\2\2\u01c0\u01c2"+ + "\5\26\f\2\u01c1\u01c0\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\3\2\2\2"+ + "\u01c3\u01c4\7N\2\2\u01c4\63\3\2\2\2\u01c5\u01ca\5\66\34\2\u01c6\u01ca"+ + "\58\35\2\u01c7\u01ca\5:\36\2\u01c8\u01ca\5<\37\2\u01c9\u01c5\3\2\2\2\u01c9"+ + "\u01c6\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01c8\3\2\2\2\u01ca\65\3\2\2"+ + "\2\u01cb\u01cc\7M\2\2\u01cc\u01cd\7\64\2\2\u01cd\u01ce\7N\2\2\u01ce\67"+ + "\3\2\2\2\u01cf\u01d0\5\26\f\2\u01d0\u01d1\7\64\2\2\u01d1\u01d2\7\26\2"+ + "\2\u01d29\3\2\2\2\u01d3\u01d4\7N\2\2\u01d4\u01d5\7\64\2\2\u01d5\u01d6"+ + "\7N\2\2\u01d6;\3\2\2\2\u01d7\u01d8\7\32\2\2\u01d8\u01d9\7\64\2\2\u01d9"+ + "\u01da\7N\2\2\u01da=\3\2\2\2*AGZ]iq~\u0082\u0086\u008b\u00a6\u00af\u00b3"+ + "\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3\u012e\u0130\u0150\u0156\u015e"+ + "\u0169\u016f\u0172\u0174\u0189\u018f\u0196\u01a2\u01a5\u01ac\u01b5\u01b8"+ + "\u01bb\u01c1\u01c9"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java index f116f087c5c..4b7daa213b3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java @@ -18,6 +18,20 @@ class PainlessParserBaseVisitor extends AbstractParseTreeVisitor implement * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitSource(PainlessParser.SourceContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitFunction(PainlessParser.FunctionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitParameters(PainlessParser.ParametersContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -151,13 +165,6 @@ class PainlessParserBaseVisitor extends AbstractParseTreeVisitor implement * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitDecltype(PainlessParser.DecltypeContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -326,6 +333,13 @@ class PainlessParserBaseVisitor extends AbstractParseTreeVisitor implement * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitString(PainlessParser.StringContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitRegex(PainlessParser.RegexContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -333,6 +347,13 @@ class PainlessParserBaseVisitor extends AbstractParseTreeVisitor implement * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitVariable(PainlessParser.VariableContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitCalllocal(PainlessParser.CalllocalContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -382,4 +403,53 @@ class PainlessParserBaseVisitor extends AbstractParseTreeVisitor implement * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitArgument(PainlessParser.ArgumentContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitLambda(PainlessParser.LambdaContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitLamtype(PainlessParser.LamtypeContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitClassFuncref(PainlessParser.ClassFuncrefContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitConstructorFuncref(PainlessParser.ConstructorFuncrefContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitCapturingFuncref(PainlessParser.CapturingFuncrefContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

+ */ + @Override public T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx) { return visitChildren(ctx); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java index f0943743ef8..f39bc84a7af 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java @@ -16,6 +16,18 @@ interface PainlessParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitSource(PainlessParser.SourceContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#function}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFunction(PainlessParser.FunctionContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#parameters}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitParameters(PainlessParser.ParametersContext ctx); /** * Visit a parse tree produced by the {@code if} * labeled alternative in {@link PainlessParser#statement}. @@ -142,12 +154,6 @@ interface PainlessParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitDecltype(PainlessParser.DecltypeContext ctx); - /** - * Visit a parse tree produced by {@link PainlessParser#funcref}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitFuncref(PainlessParser.FuncrefContext ctx); /** * Visit a parse tree produced by {@link PainlessParser#declvar}. * @param ctx the parse tree @@ -313,6 +319,13 @@ interface PainlessParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitString(PainlessParser.StringContext ctx); + /** + * Visit a parse tree produced by the {@code regex} + * labeled alternative in {@link PainlessParser#primary}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitRegex(PainlessParser.RegexContext ctx); /** * Visit a parse tree produced by the {@code variable} * labeled alternative in {@link PainlessParser#primary}. @@ -320,6 +333,13 @@ interface PainlessParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitVariable(PainlessParser.VariableContext ctx); + /** + * Visit a parse tree produced by the {@code calllocal} + * labeled alternative in {@link PainlessParser#primary}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCalllocal(PainlessParser.CalllocalContext ctx); /** * Visit a parse tree produced by the {@code newobject} * labeled alternative in {@link PainlessParser#primary}. @@ -366,4 +386,46 @@ interface PainlessParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitArgument(PainlessParser.ArgumentContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#lambda}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitLambda(PainlessParser.LambdaContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#lamtype}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitLamtype(PainlessParser.LamtypeContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#funcref}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFuncref(PainlessParser.FuncrefContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#classFuncref}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitClassFuncref(PainlessParser.ClassFuncrefContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#constructorFuncref}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitConstructorFuncref(PainlessParser.ConstructorFuncrefContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#capturingFuncref}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCapturingFuncref(PainlessParser.CapturingFuncrefContext ctx); + /** + * Visit a parse tree produced by {@link PainlessParser#localFuncref}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java new file mode 100644 index 00000000000..720259d5316 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.antlr; + +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenFactory; + +/** + * Utility to figure out if a {@code /} is division or the start of a regex literal. + */ +public class SlashStrategy { + public static boolean slashIsRegex(TokenFactory factory) { + StashingTokenFactory stashingFactory = (StashingTokenFactory) factory; + Token lastToken = stashingFactory.getLastToken(); + if (lastToken == null) { + return true; + } + switch (lastToken.getType()) { + case PainlessLexer.RBRACE: + case PainlessLexer.RP: + case PainlessLexer.OCTAL: + case PainlessLexer.HEX: + case PainlessLexer.INTEGER: + case PainlessLexer.DECIMAL: + case PainlessLexer.ID: + case PainlessLexer.DOTINTEGER: + case PainlessLexer.DOTID: + return false; + default: + return true; + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java new file mode 100644 index 00000000000..4f22137e5fb --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.antlr; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenFactory; +import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.misc.Pair; + +/** + * Token factory that preseres that last non-whitespace token so you can do token level lookbehind in the lexer. + */ +public class StashingTokenFactory implements TokenFactory { + private final TokenFactory delegate; + + private T lastToken; + + public StashingTokenFactory(TokenFactory delegate) { + this.delegate = delegate; + } + + public T getLastToken() { + return lastToken; + } + + @Override + public T create(Pair source, int type, String text, int channel, int start, int stop, int line, + int charPositionInLine) { + return maybeStash(delegate.create(source, type, text, channel, start, stop, line, charPositionInLine)); + } + + @Override + public T create(int type, String text) { + return maybeStash(delegate.create(type, text)); + } + + private T maybeStash(T token) { + if (token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL) { + lastToken = token; + } + return token; + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index 471cb7d64ea..0f6f7c8f2d5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -27,10 +27,13 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.painless.CompilerSettings; +import org.elasticsearch.painless.Locals.ExecuteReserved; +import org.elasticsearch.painless.Locals.FunctionReserved; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables.Reserved; +import org.elasticsearch.painless.Locals.Reserved; import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext; import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext; import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext; @@ -41,10 +44,14 @@ import org.elasticsearch.painless.antlr.PainlessParser.BoolContext; import org.elasticsearch.painless.antlr.PainlessParser.BraceaccessContext; import org.elasticsearch.painless.antlr.PainlessParser.BreakContext; import org.elasticsearch.painless.antlr.PainlessParser.CallinvokeContext; +import org.elasticsearch.painless.antlr.PainlessParser.CalllocalContext; +import org.elasticsearch.painless.antlr.PainlessParser.CapturingFuncrefContext; import org.elasticsearch.painless.antlr.PainlessParser.CastContext; import org.elasticsearch.painless.antlr.PainlessParser.ChainprecContext; +import org.elasticsearch.painless.antlr.PainlessParser.ClassFuncrefContext; import org.elasticsearch.painless.antlr.PainlessParser.CompContext; import org.elasticsearch.painless.antlr.PainlessParser.ConditionalContext; +import org.elasticsearch.painless.antlr.PainlessParser.ConstructorFuncrefContext; import org.elasticsearch.painless.antlr.PainlessParser.ContinueContext; import org.elasticsearch.painless.antlr.PainlessParser.DeclContext; import org.elasticsearch.painless.antlr.PainlessParser.DeclarationContext; @@ -62,16 +69,22 @@ import org.elasticsearch.painless.antlr.PainlessParser.FalseContext; import org.elasticsearch.painless.antlr.PainlessParser.FieldaccessContext; import org.elasticsearch.painless.antlr.PainlessParser.ForContext; import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext; +import org.elasticsearch.painless.antlr.PainlessParser.FunctionContext; import org.elasticsearch.painless.antlr.PainlessParser.IfContext; import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext; +import org.elasticsearch.painless.antlr.PainlessParser.LambdaContext; +import org.elasticsearch.painless.antlr.PainlessParser.LamtypeContext; +import org.elasticsearch.painless.antlr.PainlessParser.LocalFuncrefContext; import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext; import org.elasticsearch.painless.antlr.PainlessParser.NewobjectContext; import org.elasticsearch.painless.antlr.PainlessParser.NullContext; import org.elasticsearch.painless.antlr.PainlessParser.NumericContext; import org.elasticsearch.painless.antlr.PainlessParser.OperatorContext; +import org.elasticsearch.painless.antlr.PainlessParser.ParametersContext; import org.elasticsearch.painless.antlr.PainlessParser.PostContext; import org.elasticsearch.painless.antlr.PainlessParser.PreContext; import org.elasticsearch.painless.antlr.PainlessParser.ReadContext; +import org.elasticsearch.painless.antlr.PainlessParser.RegexContext; import org.elasticsearch.painless.antlr.PainlessParser.ReturnContext; import org.elasticsearch.painless.antlr.PainlessParser.SecondaryContext; import org.elasticsearch.painless.antlr.PainlessParser.SingleContext; @@ -94,21 +107,25 @@ import org.elasticsearch.painless.node.AStatement; import org.elasticsearch.painless.node.EBinary; import org.elasticsearch.painless.node.EBool; import org.elasticsearch.painless.node.EBoolean; +import org.elasticsearch.painless.node.ECapturingFunctionRef; import org.elasticsearch.painless.node.EChain; import org.elasticsearch.painless.node.EComp; import org.elasticsearch.painless.node.EConditional; import org.elasticsearch.painless.node.EDecimal; import org.elasticsearch.painless.node.EExplicit; import org.elasticsearch.painless.node.EFunctionRef; +import org.elasticsearch.painless.node.ELambda; import org.elasticsearch.painless.node.ENull; import org.elasticsearch.painless.node.ENumeric; import org.elasticsearch.painless.node.EUnary; import org.elasticsearch.painless.node.LBrace; -import org.elasticsearch.painless.node.LCall; +import org.elasticsearch.painless.node.LCallInvoke; +import org.elasticsearch.painless.node.LCallLocal; import org.elasticsearch.painless.node.LCast; import org.elasticsearch.painless.node.LField; import org.elasticsearch.painless.node.LNewArray; import org.elasticsearch.painless.node.LNewObj; +import org.elasticsearch.painless.node.LRegex; import org.elasticsearch.painless.node.LStatic; import org.elasticsearch.painless.node.LString; import org.elasticsearch.painless.node.LVariable; @@ -122,6 +139,7 @@ import org.elasticsearch.painless.node.SDo; import org.elasticsearch.painless.node.SEach; import org.elasticsearch.painless.node.SExpression; import org.elasticsearch.painless.node.SFor; +import org.elasticsearch.painless.node.SFunction; import org.elasticsearch.painless.node.SIf; import org.elasticsearch.painless.node.SIfElse; import org.elasticsearch.painless.node.SReturn; @@ -130,7 +148,10 @@ import org.elasticsearch.painless.node.SThrow; import org.elasticsearch.painless.node.STry; import org.elasticsearch.painless.node.SWhile; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; import java.util.List; /** @@ -138,19 +159,22 @@ import java.util.List; */ public final class Walker extends PainlessParserBaseVisitor { - public static SSource buildPainlessTree(String name, String sourceText, Reserved reserved, CompilerSettings settings) { - return new Walker(name, sourceText, reserved, settings).source; + public static SSource buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings) { + return new Walker(sourceName, sourceText, settings).source; } - private final Reserved reserved; private final SSource source; private final CompilerSettings settings; private final String sourceName; + private final String sourceText; - private Walker(String name, String sourceText, Reserved reserved, CompilerSettings settings) { - this.reserved = reserved; + private final Deque reserved = new ArrayDeque<>(); + private final List synthetic = new ArrayList<>(); + + private Walker(String sourceName, String sourceText, CompilerSettings settings) { this.settings = settings; - this.sourceName = Location.computeSourceName(name, sourceText); + this.sourceName = Location.computeSourceName(sourceName, sourceText); + this.sourceText = sourceText; this.source = (SSource)visit(buildAntlrTree(sourceText)); } @@ -196,13 +220,53 @@ public final class Walker extends PainlessParserBaseVisitor { @Override public Object visitSource(SourceContext ctx) { + reserved.push(new ExecuteReserved()); + + List functions = new ArrayList<>(); + + for (FunctionContext function : ctx.function()) { + functions.add((SFunction)visit(function)); + } + List statements = new ArrayList<>(); for (StatementContext statement : ctx.statement()) { statements.add((AStatement)visit(statement)); } + + functions.addAll(synthetic); - return new SSource(location(ctx), statements); + return new SSource(sourceName, sourceText, (ExecuteReserved)reserved.pop(), location(ctx), functions, statements); + } + + @Override + public Object visitFunction(FunctionContext ctx) { + reserved.push(new FunctionReserved()); + + String rtnType = ctx.decltype().getText(); + String name = ctx.ID().getText(); + List paramTypes = new ArrayList<>(); + List paramNames = new ArrayList<>(); + List statements = new ArrayList<>(); + + for (DecltypeContext decltype : ctx.parameters().decltype()) { + paramTypes.add(decltype.getText()); + } + + for (TerminalNode id : ctx.parameters().ID()) { + paramNames.add(id.getText()); + } + + for (StatementContext statement : ctx.block().statement()) { + statements.add((AStatement)visit(statement)); + } + + return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, paramTypes, paramNames, statements, false); + } + + @Override + public Object visitParameters(ParametersContext ctx) { + throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); } @Override @@ -221,18 +285,16 @@ public final class Walker extends PainlessParserBaseVisitor { @Override public Object visitWhile(WhileContext ctx) { - if (settings.getMaxLoopCounter() > 0) { - reserved.usesLoop(); - } + reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter()); AExpression expression = (AExpression)visitExpression(ctx.expression()); if (ctx.trailer() != null) { SBlock block = (SBlock)visit(ctx.trailer()); - return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, block); + return new SWhile(location(ctx), expression, block); } else if (ctx.empty() != null) { - return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, null); + return new SWhile(location(ctx), expression, null); } else { throw location(ctx).createError(new IllegalStateException(" Illegal tree structure.")); } @@ -240,21 +302,17 @@ public final class Walker extends PainlessParserBaseVisitor { @Override public Object visitDo(DoContext ctx) { - if (settings.getMaxLoopCounter() > 0) { - reserved.usesLoop(); - } + reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter()); AExpression expression = (AExpression)visitExpression(ctx.expression()); SBlock block = (SBlock)visit(ctx.block()); - return new SDo(location(ctx), settings.getMaxLoopCounter(), block, expression); + return new SDo(location(ctx), block, expression); } @Override public Object visitFor(ForContext ctx) { - if (settings.getMaxLoopCounter() > 0) { - reserved.usesLoop(); - } + reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter()); ANode initializer = ctx.initializer() == null ? null : (ANode)visit(ctx.initializer()); AExpression expression = ctx.expression() == null ? null : (AExpression)visitExpression(ctx.expression()); @@ -263,9 +321,9 @@ public final class Walker extends PainlessParserBaseVisitor { if (ctx.trailer() != null) { SBlock block = (SBlock)visit(ctx.trailer()); - return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, block); + return new SFor(location(ctx), initializer, expression, afterthought, block); } else if (ctx.empty() != null) { - return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, null); + return new SFor(location(ctx), initializer, expression, afterthought, null); } else { throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); } @@ -273,16 +331,14 @@ public final class Walker extends PainlessParserBaseVisitor { @Override public Object visitEach(EachContext ctx) { - if (settings.getMaxLoopCounter() > 0) { - reserved.usesLoop(); - } + reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter()); String type = ctx.decltype().getText(); String name = ctx.ID().getText(); AExpression expression = (AExpression)visitExpression(ctx.expression()); SBlock block = (SBlock)visit(ctx.trailer()); - return new SEach(location(ctx), settings.getMaxLoopCounter(), type, name, expression, block); + return new SEach(location(ctx), type, name, expression, block); } @Override @@ -403,19 +459,6 @@ public final class Walker extends PainlessParserBaseVisitor { throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); } - @Override - public Object visitFuncref(FuncrefContext ctx) { - final String methodText; - if (ctx.ID() != null) { - methodText = ctx.ID().getText(); - } else if (ctx.NEW() != null ){ - methodText = ctx.NEW().getText(); - } else { - throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); - } - return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText); - } - @Override public Object visitDeclvar(DeclvarContext ctx) { throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); @@ -785,13 +828,33 @@ public final class Walker extends PainlessParserBaseVisitor { return links; } + @Override + public Object visitRegex(RegexContext ctx) { + String pattern = ctx.REGEX().getText().substring(1, ctx.REGEX().getText().length() - 1); + List links = new ArrayList<>(); + links.add(new LRegex(location(ctx), pattern)); + + return links; + } + @Override public Object visitVariable(VariableContext ctx) { String name = ctx.ID().getText(); List links = new ArrayList<>(); links.add(new LVariable(location(ctx), name)); - reserved.markReserved(name); + reserved.peek().markReserved(name); + + return links; + } + + @Override + public Object visitCalllocal(CalllocalContext ctx) { + String name = ctx.ID().getText(); + @SuppressWarnings("unchecked") + List arguments = (List)visit(ctx.arguments()); + List links = new ArrayList<>(); + links.add(new LCallLocal(location(ctx), name, arguments)); return links; } @@ -825,7 +888,7 @@ public final class Walker extends PainlessParserBaseVisitor { @SuppressWarnings("unchecked") List arguments = (List)visit(ctx.arguments()); - return new LCall(location(ctx), name, arguments); + return new LCallInvoke(location(ctx), name, arguments); } @Override @@ -865,10 +928,92 @@ public final class Walker extends PainlessParserBaseVisitor { public Object visitArgument(ArgumentContext ctx) { if (ctx.expression() != null) { return visitExpression(ctx.expression()); + } else if (ctx.lambda() != null) { + return visit(ctx.lambda()); } else if (ctx.funcref() != null) { return visit(ctx.funcref()); } else { throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); } } + + @Override + public Object visitLambda(LambdaContext ctx) { + reserved.push(new FunctionReserved()); + + List paramTypes = new ArrayList<>(); + List paramNames = new ArrayList<>(); + List statements = new ArrayList<>(); + + for (LamtypeContext lamtype : ctx.lamtype()) { + if (lamtype.decltype() == null) { + paramTypes.add(null); + } else { + paramTypes.add(lamtype.decltype().getText()); + } + + paramNames.add(lamtype.ID().getText()); + } + + for (StatementContext statement : ctx.block().statement()) { + statements.add((AStatement)visit(statement)); + } + + return new ELambda((FunctionReserved)reserved.pop(), location(ctx), paramTypes, paramNames, statements); + } + + @Override + public Object visitLamtype(LamtypeContext ctx) { + throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); + } + + @Override + public Object visitFuncref(FuncrefContext ctx) { + if (ctx.classFuncref() != null) { + return visit(ctx.classFuncref()); + } else if (ctx.constructorFuncref() != null) { + return visit(ctx.constructorFuncref()); + } else if (ctx.capturingFuncref() != null) { + return visit(ctx.capturingFuncref()); + } else if (ctx.localFuncref() != null) { + return visit(ctx.localFuncref()); + } else { + throw location(ctx).createError(new IllegalStateException("Illegal tree structure.")); + } + } + + @Override + public Object visitClassFuncref(ClassFuncrefContext ctx) { + return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.ID().getText()); + } + + @Override + public Object visitConstructorFuncref(ConstructorFuncrefContext ctx) { + if (!ctx.decltype().LBRACE().isEmpty()) { + // array constructors are special: we need to make a synthetic method + // taking integer as argument and returning a new instance, and return a ref to that. + Location location = location(ctx); + String arrayType = ctx.decltype().getText(); + SReturn code = new SReturn(location, + new EChain(location, + new LNewArray(location, arrayType, Arrays.asList( + new EChain(location, + new LVariable(location, "size")))))); + String name = "lambda$" + synthetic.size(); + synthetic.add(new SFunction(new FunctionReserved(), location, arrayType, name, + Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true)); + return new EFunctionRef(location(ctx), "this", name); + } + return new EFunctionRef(location(ctx), ctx.decltype().getText(), ctx.NEW().getText()); + } + + @Override + public Object visitCapturingFuncref(CapturingFuncrefContext ctx) { + return new ECapturingFunctionRef(location(ctx), ctx.ID(0).getText(), ctx.ID(1).getText()); + } + + @Override + public Object visitLocalFuncref(LocalFuncrefContext ctx) { + return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID().getText()); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java index c5494b978d8..fbaaa83a6ad 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java @@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -106,7 +106,7 @@ public abstract class AExpression extends ANode { /** * Checks for errors and collects data for the writing phase. */ - abstract void analyze(Variables variables); + abstract void analyze(Locals locals); /** * Writes ASM based on the data collected during the analysis phase. @@ -118,7 +118,7 @@ public abstract class AExpression extends ANode { * nodes with the constant variable set to a non-null value with {@link EConstant}. * @return The new child node for the parent node calling this method. */ - AExpression cast(Variables variables) { + AExpression cast(Locals locals) { final Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal); if (cast == null) { @@ -136,7 +136,7 @@ public abstract class AExpression extends ANode { // will already be the same. EConstant econstant = new EConstant(location, constant); - econstant.analyze(variables); + econstant.analyze(locals); if (!expected.equals(econstant.actual)) { throw createError(new IllegalStateException("Illegal tree structure.")); @@ -170,7 +170,7 @@ public abstract class AExpression extends ANode { constant = AnalyzerCaster.constCast(location, constant, cast); EConstant econstant = new EConstant(location, constant); - econstant.analyze(variables); + econstant.analyze(locals); if (!expected.equals(econstant.actual)) { throw createError(new IllegalStateException("Illegal tree structure.")); @@ -201,7 +201,7 @@ public abstract class AExpression extends ANode { // the EConstant will already be the same. EConstant econstant = new EConstant(location, constant); - econstant.analyze(variables); + econstant.analyze(locals); if (!actual.equals(econstant.actual)) { throw createError(new IllegalStateException("Illegal tree structure.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java index 19875ebeec5..6d52bfe0de1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -86,7 +86,7 @@ public abstract class ALink extends ANode { * def or a shortcut is used. Otherwise, returns itself. This will be * updated into the {@link EChain} node's list of links. */ - abstract ALink analyze(Variables variables); + abstract ALink analyze(Locals locals); /** * Write values before a load/store occurs such as an array index. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java index b9e6679f630..e34d174ac43 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -115,7 +115,7 @@ public abstract class AStatement extends ANode { /** * Checks for errors and collects data for the writing phase. */ - abstract void analyze(Variables variables); + abstract void analyze(Locals locals); /** * Writes ASM based on the data collected during the analysis phase. diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java index 32a6e2382f2..90d7b819902 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java @@ -26,7 +26,7 @@ import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; /** * Represents a binary math expression. @@ -36,6 +36,8 @@ public final class EBinary extends AExpression { final Operation operation; AExpression left; AExpression right; + Type promote; // promoted type + Type shiftDistance; // for shifts, the RHS is promoted independently boolean cat = false; @@ -48,47 +50,57 @@ public final class EBinary extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (operation == Operation.MUL) { - analyzeMul(variables); + analyzeMul(locals); } else if (operation == Operation.DIV) { - analyzeDiv(variables); + analyzeDiv(locals); } else if (operation == Operation.REM) { - analyzeRem(variables); + analyzeRem(locals); } else if (operation == Operation.ADD) { - analyzeAdd(variables); + analyzeAdd(locals); } else if (operation == Operation.SUB) { - analyzeSub(variables); + analyzeSub(locals); } else if (operation == Operation.LSH) { - analyzeLSH(variables); + analyzeLSH(locals); } else if (operation == Operation.RSH) { - analyzeRSH(variables); + analyzeRSH(locals); } else if (operation == Operation.USH) { - analyzeUSH(variables); + analyzeUSH(locals); } else if (operation == Operation.BWAND) { - analyzeBWAnd(variables); + analyzeBWAnd(locals); } else if (operation == Operation.XOR) { - analyzeXor(variables); + analyzeXor(locals); } else if (operation == Operation.BWOR) { - analyzeBWOr(variables); + analyzeBWOr(locals); } else { throw createError(new IllegalStateException("Illegal tree structure.")); } } - private void analyzeMul(Variables variables) { + private void analyzeMul(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply multiply [*] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -108,23 +120,30 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeDiv(Variables variables) { + private void analyzeDiv(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply divide [/] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -148,23 +167,31 @@ public final class EBinary extends AExpression { throw createError(e); } } - - actual = promote; } - private void analyzeRem(Variables variables) { + private void analyzeRem(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply remainder [%] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -188,15 +215,13 @@ public final class EBinary extends AExpression { throw createError(e); } } - - actual = promote; } - private void analyzeAdd(Variables variables) { + private void analyzeAdd(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual); + promote = AnalyzerCaster.promoteAdd(left.actual, right.actual); if (promote == null) { throw createError(new ClassCastException("Cannot apply add [+] to types " + @@ -205,6 +230,8 @@ public final class EBinary extends AExpression { Sort sort = promote.sort; + actual = promote; + if (sort == Sort.STRING) { left.expected = left.actual; @@ -217,6 +244,12 @@ public final class EBinary extends AExpression { if (right instanceof EBinary && ((EBinary)right).operation == Operation.ADD && right.actual.sort == Sort.STRING) { ((EBinary)right).cat = true; } + } else if (sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } } else { left.expected = promote; right.expected = promote; @@ -241,22 +274,31 @@ public final class EBinary extends AExpression { } } - actual = promote; } - private void analyzeSub(Variables variables) { + private void analyzeSub(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply subtract [-] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -276,30 +318,44 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeLSH(Variables variables) { + private void analyzeLSH(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); - if (promote == null) { + if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply left shift [<<] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = Definition.INT_TYPE; - right.explicit = true; + actual = promote = lhspromote; + shiftDistance = rhspromote; + + if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = lhspromote; + if (rhspromote.sort == Sort.LONG) { + right.expected = Definition.INT_TYPE; + right.explicit = true; + } else { + right.expected = rhspromote; + } + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = lhspromote.sort; if (sort == Sort.INT) { constant = (int)left.constant << (int)right.constant; @@ -309,30 +365,44 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeRSH(Variables variables) { + private void analyzeRSH(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); - if (promote == null) { + if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply right shift [>>] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = Definition.INT_TYPE; - right.explicit = true; + actual = promote = lhspromote; + shiftDistance = rhspromote; + + if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = lhspromote; + if (rhspromote.sort == Sort.LONG) { + right.expected = Definition.INT_TYPE; + right.explicit = true; + } else { + right.expected = rhspromote; + } + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = lhspromote.sort; if (sort == Sort.INT) { constant = (int)left.constant >> (int)right.constant; @@ -342,30 +412,44 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeUSH(Variables variables) { + private void analyzeUSH(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false); + Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false); - if (promote == null) { + actual = promote = lhspromote; + shiftDistance = rhspromote; + + if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = Definition.INT_TYPE; - right.explicit = true; + if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = lhspromote; + if (rhspromote.sort == Sort.LONG) { + right.expected = Definition.INT_TYPE; + right.explicit = true; + } else { + right.expected = rhspromote; + } + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = lhspromote.sort; if (sort == Sort.INT) { constant = (int)left.constant >>> (int)right.constant; @@ -375,23 +459,31 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeBWAnd(Variables variables) { + private void analyzeBWAnd(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply and [&] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -407,23 +499,31 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeXor(Variables variables) { + private void analyzeXor(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual); + promote = AnalyzerCaster.promoteXor(left.actual, right.actual); if (promote == null) { throw createError(new ClassCastException("Cannot apply xor [^] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -441,23 +541,31 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } - private void analyzeBWOr(Variables variables) { + private void analyzeBWOr(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); + promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply or [|] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + actual = promote; + + if (promote.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + if (expected != null) { + actual = expected; + } + } else { + left.expected = promote; + right.expected = promote; + } left = left.cast(variables); right = right.cast(variables); @@ -473,15 +581,13 @@ public final class EBinary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } } - - actual = promote; } @Override void write(MethodWriter writer) { writer.writeDebugInfo(location); - if (actual.sort == Sort.STRING && operation == Operation.ADD) { + if (promote.sort == Sort.STRING && operation == Operation.ADD) { if (!cat) { writer.writeNewStrings(); } @@ -505,7 +611,11 @@ public final class EBinary extends AExpression { left.write(writer); right.write(writer); - writer.writeBinaryInstruction(location, actual, operation); + if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) { + writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation); + } else { + writer.writeBinaryInstruction(location, actual, operation); + } } writer.writeBranch(tru, fals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java index dd221f040c4..5aa2daeeb34 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -44,14 +44,14 @@ public final class EBool extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { left.expected = Definition.BOOLEAN_TYPE; - left.analyze(variables); - left = left.cast(variables); + left.analyze(locals); + left = left.cast(locals); right.expected = Definition.BOOLEAN_TYPE; - right.analyze(variables); - right = right.cast(variables); + right.analyze(locals); + right = right.cast(locals); if (left.constant != null && right.constant != null) { if (operation == Operation.AND) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java index 5065989600f..877e79549af 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -36,7 +36,7 @@ public final class EBoolean extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { actual = Definition.BOOLEAN_TYPE; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java new file mode 100644 index 00000000000..ac316f07fbd --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.node; + +import org.elasticsearch.painless.DefBootstrap; +import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.FunctionRef; +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Variable; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE; +import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; + +import java.lang.invoke.LambdaMetafactory; + +/** + * Represents a capturing function reference. + */ +public class ECapturingFunctionRef extends AExpression { + public final String type; + public final String call; + + private FunctionRef ref; + Variable captured; + private boolean defInterface; + + public ECapturingFunctionRef(Location location, String type, String call) { + super(location); + + this.type = type; + this.call = call; + } + + @Override + void analyze(Locals variables) { + captured = variables.getVariable(location, type); + if (expected == null) { + defInterface = true; + actual = Definition.getType("String"); + } else { + defInterface = false; + // static case + if (captured.type.sort != Definition.Sort.DEF) { + try { + ref = new FunctionRef(expected, captured.type.name, call, captured.type.clazz); + } catch (IllegalArgumentException e) { + throw createError(e); + } + } + actual = expected; + } + } + + @Override + void write(MethodWriter writer) { + writer.writeDebugInfo(location); + if (defInterface && captured.type.sort == Definition.Sort.DEF) { + // dynamic interface, dynamic implementation + writer.push("D" + type + "." + call + ",1"); + writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot); + } else if (defInterface) { + // dynamic interface, typed implementation + writer.push("S" + captured.type.name + "." + call + ",1"); + writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot); + } else if (ref == null) { + // typed interface, dynamic implementation + writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot); + String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor(); + writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.REFERENCE, expected.name); + } else { + // typed interface, typed implementation + writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot); + // convert MethodTypes to asm Type for the constant pool. + String invokedType = ref.invokedType.toMethodDescriptorString(); + Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString()); + Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString()); + if (ref.needsBridges()) { + writer.invokeDynamic(ref.invokedName, + invokedType, + LAMBDA_BOOTSTRAP_HANDLE, + samMethodType, + ref.implMethodASM, + samMethodType, + LambdaMetafactory.FLAG_BRIDGES, + 1, + interfaceType); + } else { + writer.invokeDynamic(ref.invokedName, + invokedType, + LAMBDA_BOOTSTRAP_HANDLE, + samMethodType, + ref.implMethodASM, + samMethodType, + 0); + } + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java index 27974240125..7892b918ed1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -45,7 +45,7 @@ final class ECast extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { throw createError(new IllegalStateException("Illegal tree structure.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java index 9b2968db565..2d47c520b7b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java @@ -26,9 +26,10 @@ import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; +import java.util.Arrays; import java.util.List; /** @@ -46,6 +47,11 @@ public final class EChain extends AExpression { Type promote = null; Cast there = null; Cast back = null; + + /** Creates a new RHS-only EChain */ + public EChain(Location location, ALink link) { + this(location, Arrays.asList(link), false, false, null, null); + } public EChain(Location location, List links, boolean pre, boolean post, Operation operation, AExpression expression) { @@ -59,20 +65,20 @@ public final class EChain extends AExpression { } @Override - void analyze(Variables variables) { - analyzeLinks(variables); + void analyze(Locals locals) { + analyzeLinks(locals); analyzeIncrDecr(); if (operation != null) { - analyzeCompound(variables); + analyzeCompound(locals); } else if (expression != null) { - analyzeWrite(variables); + analyzeWrite(locals); } else { analyzeRead(); } } - private void analyzeLinks(Variables variables) { + private void analyzeLinks(Locals variables) { ALink previous = null; int index = 0; @@ -153,7 +159,7 @@ public final class EChain extends AExpression { } } - private void analyzeCompound(Variables variables) { + private void analyzeCompound(Locals variables) { ALink last = links.get(links.size() - 1); expression.analyze(variables); @@ -214,7 +220,7 @@ public final class EChain extends AExpression { this.actual = read ? last.after : Definition.VOID_TYPE; } - private void analyzeWrite(Variables variables) { + private void analyzeWrite(Locals variables) { ALink last = links.get(links.size() - 1); // If the store node is a def node, we remove the cast to def from the expression @@ -328,7 +334,15 @@ public final class EChain extends AExpression { writer.writeCast(there); // if necessary cast the current link's value // to the promotion type between the lhs and rhs types expression.write(writer); // write the bytecode for the rhs expression - writer.writeBinaryInstruction(location, promote, operation); // write the operation instruction for compound assignment + // XXX: fix these types, but first we need def compound assignment tests. + // (and also corner cases such as shifts). its tricky here as there are possibly explicit casts, too. + // write the operation instruction for compound assignment + if (promote.sort == Sort.DEF) { + writer.writeDynamicBinaryInstruction(location, promote, + Definition.DEF_TYPE, Definition.DEF_TYPE, operation); + } else { + writer.writeBinaryInstruction(location, promote, operation); + } writer.writeCast(back); // if necessary cast the promotion type value back to the link's type diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java index 407b59f92cf..1f4f691d4aa 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java @@ -24,19 +24,15 @@ import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.AnalyzerCaster; +import org.elasticsearch.painless.DefBootstrap; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; -import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS; -import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE; -import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE; +import static org.elasticsearch.painless.WriterConstants.OBJECTS_TYPE; +import static org.elasticsearch.painless.WriterConstants.EQUALS; +import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE; /** * Represents a comparison expression. @@ -46,6 +42,7 @@ public final class EComp extends AExpression { final Operation operation; AExpression left; AExpression right; + Type promotedType; public EComp(Location location, Operation operation, AExpression left, AExpression right) { super(location); @@ -56,41 +53,46 @@ public final class EComp extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (operation == Operation.EQ) { - analyzeEq(variables); + analyzeEq(locals); } else if (operation == Operation.EQR) { - analyzeEqR(variables); + analyzeEqR(locals); } else if (operation == Operation.NE) { - analyzeNE(variables); + analyzeNE(locals); } else if (operation == Operation.NER) { - analyzeNER(variables); + analyzeNER(locals); } else if (operation == Operation.GTE) { - analyzeGTE(variables); + analyzeGTE(locals); } else if (operation == Operation.GT) { - analyzeGT(variables); + analyzeGT(locals); } else if (operation == Operation.LTE) { - analyzeLTE(variables); + analyzeLTE(locals); } else if (operation == Operation.LT) { - analyzeLT(variables); + analyzeLT(locals); } else { throw createError(new IllegalStateException("Illegal tree structure.")); } } - private void analyzeEq(Variables variables) { + private void analyzeEq(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply equals [==] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); @@ -100,7 +102,7 @@ public final class EComp extends AExpression { } if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.BOOL) { constant = (boolean)left.constant == (boolean)right.constant; @@ -124,19 +126,24 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeEqR(Variables variables) { + private void analyzeEqR(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference equals [===] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); @@ -146,7 +153,7 @@ public final class EComp extends AExpression { } if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.BOOL) { constant = (boolean)left.constant == (boolean)right.constant; @@ -166,19 +173,24 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeNE(Variables variables) { + private void analyzeNE(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply not equals [!=] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); @@ -188,7 +200,7 @@ public final class EComp extends AExpression { } if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.BOOL) { constant = (boolean)left.constant != (boolean)right.constant; @@ -212,19 +224,24 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeNER(Variables variables) { + private void analyzeNER(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual); + promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); @@ -234,7 +251,7 @@ public final class EComp extends AExpression { } if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.BOOL) { constant = (boolean)left.constant != (boolean)right.constant; @@ -254,25 +271,30 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeGTE(Variables variables) { + private void analyzeGTE(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.INT) { constant = (int)left.constant >= (int)right.constant; @@ -290,25 +312,30 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeGT(Variables variables) { + private void analyzeGT(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than [>] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.INT) { constant = (int)left.constant > (int)right.constant; @@ -326,25 +353,30 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeLTE(Variables variables) { + private void analyzeLTE(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.INT) { constant = (int)left.constant <= (int)right.constant; @@ -362,25 +394,30 @@ public final class EComp extends AExpression { actual = Definition.BOOLEAN_TYPE; } - private void analyzeLT(Variables variables) { + private void analyzeLT(Locals variables) { left.analyze(variables); right.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); + promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true); - if (promote == null) { + if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than [>=] to types " + "[" + left.actual.name + "] and [" + right.actual.name + "].")); } - left.expected = promote; - right.expected = promote; + if (promotedType.sort == Sort.DEF) { + left.expected = left.actual; + right.expected = right.actual; + } else { + left.expected = promotedType; + right.expected = promotedType; + } left = left.cast(variables); right = right.cast(variables); if (left.constant != null && right.constant != null) { - Sort sort = promote.sort; + Sort sort = promotedType.sort; if (sort == Sort.INT) { constant = (int)left.constant < (int)right.constant; @@ -403,8 +440,6 @@ public final class EComp extends AExpression { writer.writeDebugInfo(location); boolean branch = tru != null || fals != null; - org.objectweb.asm.Type rtype = right.actual.type; - Sort rsort = right.actual.sort; left.write(writer); @@ -426,7 +461,7 @@ public final class EComp extends AExpression { boolean writejump = true; - switch (rsort) { + switch (promotedType.sort) { case VOID: case BYTE: case SHORT: @@ -444,47 +479,49 @@ public final class EComp extends AExpression { case LONG: case FLOAT: case DOUBLE: - if (eq) writer.ifCmp(rtype, MethodWriter.EQ, jump); - else if (ne) writer.ifCmp(rtype, MethodWriter.NE, jump); - else if (lt) writer.ifCmp(rtype, MethodWriter.LT, jump); - else if (lte) writer.ifCmp(rtype, MethodWriter.LE, jump); - else if (gt) writer.ifCmp(rtype, MethodWriter.GT, jump); - else if (gte) writer.ifCmp(rtype, MethodWriter.GE, jump); + if (eq) writer.ifCmp(promotedType.type, MethodWriter.EQ, jump); + else if (ne) writer.ifCmp(promotedType.type, MethodWriter.NE, jump); + else if (lt) writer.ifCmp(promotedType.type, MethodWriter.LT, jump); + else if (lte) writer.ifCmp(promotedType.type, MethodWriter.LE, jump); + else if (gt) writer.ifCmp(promotedType.type, MethodWriter.GT, jump); + else if (gte) writer.ifCmp(promotedType.type, MethodWriter.GE, jump); else { throw createError(new IllegalStateException("Illegal tree structure.")); } break; case DEF: + org.objectweb.asm.Type booleanType = org.objectweb.asm.Type.getType(boolean.class); + org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(booleanType, left.actual.type, right.actual.type); if (eq) { if (right.isNull) { writer.ifNull(jump); } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL); + writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writejump = false; } else { - writer.ifCmp(rtype, MethodWriter.EQ, jump); + writer.ifCmp(promotedType.type, MethodWriter.EQ, jump); } } else if (ne) { if (right.isNull) { writer.ifNonNull(jump); } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL); + writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writer.ifZCmp(MethodWriter.EQ, jump); } else { - writer.ifCmp(rtype, MethodWriter.NE, jump); + writer.ifCmp(promotedType.type, MethodWriter.NE, jump); } } else if (lt) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_LT_CALL); + writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writejump = false; } else if (lte) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_LTE_CALL); + writer.invokeDynamic("lte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writejump = false; } else if (gt) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_GT_CALL); + writer.invokeDynamic("gt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writejump = false; } else if (gte) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_GTE_CALL); + writer.invokeDynamic("gte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); writejump = false; } else { throw createError(new IllegalStateException("Illegal tree structure.")); @@ -500,7 +537,7 @@ public final class EComp extends AExpression { if (right.isNull) { writer.ifNull(jump); } else if (operation == Operation.EQ || operation == Operation.NE) { - writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS); + writer.invokeStatic(OBJECTS_TYPE, EQUALS); if (branch) { writer.ifZCmp(MethodWriter.NE, jump); @@ -508,16 +545,16 @@ public final class EComp extends AExpression { writejump = false; } else { - writer.ifCmp(rtype, MethodWriter.EQ, jump); + writer.ifCmp(promotedType.type, MethodWriter.EQ, jump); } } else if (ne) { if (right.isNull) { writer.ifNonNull(jump); } else if (operation == Operation.EQ || operation == Operation.NE) { - writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS); + writer.invokeStatic(OBJECTS_TYPE, EQUALS); writer.ifZCmp(MethodWriter.EQ, jump); } else { - writer.ifCmp(rtype, MethodWriter.NE, jump); + writer.ifCmp(promotedType.type, MethodWriter.NE, jump); } } else { throw createError(new IllegalStateException("Illegal tree structure.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java index 65231c45ed3..024e0c800fc 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java @@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -45,10 +45,10 @@ public final class EConditional extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); if (condition.constant != null) { throw createError(new IllegalArgumentException("Extraneous conditional statement.")); @@ -62,8 +62,8 @@ public final class EConditional extends AExpression { right.internal = internal; actual = expected; - left.analyze(variables); - right.analyze(variables); + left.analyze(locals); + right.analyze(locals); if (expected == null) { final Type promote = AnalyzerCaster.promoteConditional(left.actual, right.actual, left.constant, right.constant); @@ -73,8 +73,8 @@ public final class EConditional extends AExpression { actual = promote; } - left = left.cast(variables); - right = right.cast(variables); + left = left.cast(locals); + right = right.cast(locals); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java index 8cb032c5c3f..267d32983b9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -38,7 +38,7 @@ final class EConstant extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (constant instanceof String) { actual = Definition.STRING_TYPE; } else if (constant instanceof Double) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java index e59002c2a3b..0fd7fe46b51 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -38,7 +38,7 @@ public final class EDecimal extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (value.endsWith("f") || value.endsWith("F")) { try { constant = Float.parseFloat(value.substring(0, value.length() - 1)); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java index 0a55727b5e9..007a5d59e59 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -40,7 +40,7 @@ public final class EExplicit extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { try { actual = Definition.getType(this.type); } catch (IllegalArgumentException exception) { @@ -49,8 +49,8 @@ public final class EExplicit extends AExpression { child.expected = actual; child.explicit = true; - child.analyze(variables); - child = child.cast(variables); + child.analyze(locals); + child = child.cast(locals); } @Override @@ -58,11 +58,11 @@ public final class EExplicit extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } - AExpression cast(Variables variables) { + AExpression cast(Locals locals) { child.expected = expected; child.explicit = explicit; child.internal = internal; - return child.cast(variables); + return child.cast(locals); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index 38b7a1b8ddb..00a69e0b7b9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -23,7 +23,9 @@ import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Type; import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; @@ -36,7 +38,7 @@ import java.lang.invoke.LambdaMetafactory; public class EFunctionRef extends AExpression { public final String type; public final String call; - + private FunctionRef ref; public EFunctionRef(Location location, String type, String call) { @@ -47,13 +49,29 @@ public class EFunctionRef extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (expected == null) { ref = null; actual = Definition.getType("String"); } else { try { - ref = new FunctionRef(expected, type, call); + if ("this".equals(type)) { + // user's own function + Method interfaceMethod = expected.struct.getFunctionalMethod(); + if (interfaceMethod == null) { + throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + + "to [" + expected.name + "], not a functional interface"); + } + Method implMethod = locals.getMethod(new MethodKey(call, interfaceMethod.arguments.size())); + if (implMethod == null) { + throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + + "to [" + expected.name + "], function not found"); + } + ref = new FunctionRef(expected, interfaceMethod, implMethod); + } else { + // whitelist lookup + ref = new FunctionRef(expected, type, call); + } } catch (IllegalArgumentException e) { throw createError(e); } @@ -64,7 +82,7 @@ public class EFunctionRef extends AExpression { @Override void write(MethodWriter writer) { if (ref == null) { - writer.push(type + "." + call); + writer.push("S" + type + "." + call + ",0"); } else { writer.writeDebugInfo(location); // convert MethodTypes to asm Type for the constant pool. @@ -72,22 +90,22 @@ public class EFunctionRef extends AExpression { Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString()); Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString()); if (ref.needsBridges()) { - writer.invokeDynamic(ref.invokedName, - invokedType, - LAMBDA_BOOTSTRAP_HANDLE, - samMethodType, - ref.implMethodASM, - samMethodType, - LambdaMetafactory.FLAG_BRIDGES, - 1, + writer.invokeDynamic(ref.invokedName, + invokedType, + LAMBDA_BOOTSTRAP_HANDLE, + samMethodType, + ref.implMethodASM, + samMethodType, + LambdaMetafactory.FLAG_BRIDGES, + 1, interfaceType); } else { - writer.invokeDynamic(ref.invokedName, - invokedType, - LAMBDA_BOOTSTRAP_HANDLE, - samMethodType, - ref.implMethodASM, - samMethodType, + writer.invokeDynamic(ref.invokedName, + invokedType, + LAMBDA_BOOTSTRAP_HANDLE, + samMethodType, + ref.implMethodASM, + samMethodType, 0); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java new file mode 100644 index 00000000000..b513e3813c4 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.node; + +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.FunctionReserved; +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.MethodWriter; + +import java.util.Collections; +import java.util.List; + +public class ELambda extends AExpression { + final FunctionReserved reserved; + final List paramTypeStrs; + final List paramNameStrs; + final List statements; + + public ELambda(FunctionReserved reserved, Location location, + List paramTypes, List paramNames, List statements) { + super(location); + + this.reserved = reserved; + this.paramTypeStrs = Collections.unmodifiableList(paramTypes); + this.paramNameStrs = Collections.unmodifiableList(paramNames); + this.statements = Collections.unmodifiableList(statements); + } + + @Override + void analyze(Locals locals) { + throw createError(new UnsupportedOperationException("Lambda functions are not supported.")); + } + + @Override + void write(MethodWriter writer) { + throw createError(new IllegalStateException("Illegal tree structure.")); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java index 3f934da4968..eefc09c5946 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Opcodes; import org.elasticsearch.painless.MethodWriter; @@ -35,7 +35,7 @@ public final class ENull extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { isNull = true; if (expected != null) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java index 53f3559c294..9abdc8f6a12 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Sort; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -41,7 +41,7 @@ public final class ENumeric extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (value.endsWith("d") || value.endsWith("D")) { if (radix != 10) { throw createError(new IllegalStateException("Illegal tree structure.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java index 89d654cdf1f..bb2ff4d1d47 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java @@ -24,14 +24,13 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.AnalyzerCaster; +import org.elasticsearch.painless.DefBootstrap; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; -import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL; -import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE; +import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE; /** * Represents a unary math expression. @@ -40,6 +39,7 @@ public final class EUnary extends AExpression { final Operation operation; AExpression child; + Type promote; public EUnary(Location location, Operation operation, AExpression child) { super(location); @@ -49,21 +49,21 @@ public final class EUnary extends AExpression { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (operation == Operation.NOT) { - analyzeNot(variables); + analyzeNot(locals); } else if (operation == Operation.BWNOT) { - analyzeBWNot(variables); + analyzeBWNot(locals); } else if (operation == Operation.ADD) { - analyzerAdd(variables); + analyzerAdd(locals); } else if (operation == Operation.SUB) { - analyzerSub(variables); + analyzerSub(locals); } else { throw createError(new IllegalStateException("Illegal tree structure.")); } } - void analyzeNot(Variables variables) { + void analyzeNot(Locals variables) { child.expected = Definition.BOOLEAN_TYPE; child.analyze(variables); child = child.cast(variables); @@ -75,10 +75,10 @@ public final class EUnary extends AExpression { actual = Definition.BOOLEAN_TYPE; } - void analyzeBWNot(Variables variables) { + void analyzeBWNot(Locals variables) { child.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(child.actual, false); + promote = AnalyzerCaster.promoteNumeric(child.actual, false); if (promote == null) { throw createError(new ClassCastException("Cannot apply not [~] to type [" + child.actual.name + "].")); @@ -99,13 +99,17 @@ public final class EUnary extends AExpression { } } - actual = promote; + if (promote.sort == Sort.DEF && expected != null) { + actual = expected; + } else { + actual = promote; + } } - void analyzerAdd(Variables variables) { + void analyzerAdd(Locals variables) { child.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(child.actual, true); + promote = AnalyzerCaster.promoteNumeric(child.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply positive [+] to type [" + child.actual.name + "].")); @@ -130,13 +134,17 @@ public final class EUnary extends AExpression { } } - actual = promote; + if (promote.sort == Sort.DEF && expected != null) { + actual = expected; + } else { + actual = promote; + } } - void analyzerSub(Variables variables) { + void analyzerSub(Locals variables) { child.analyze(variables); - Type promote = AnalyzerCaster.promoteNumeric(child.actual, true); + promote = AnalyzerCaster.promoteNumeric(child.actual, true); if (promote == null) { throw createError(new ClassCastException("Cannot apply negative [-] to type [" + child.actual.name + "].")); @@ -161,7 +169,11 @@ public final class EUnary extends AExpression { } } - actual = promote; + if (promote.sort == Sort.DEF && expected != null) { + actual = expected; + } else { + actual = promote; + } } @Override @@ -187,14 +199,13 @@ public final class EUnary extends AExpression { child.write(writer); } } else { - org.objectweb.asm.Type type = actual.type; - Sort sort = actual.sort; - + Sort sort = promote.sort; child.write(writer); if (operation == Operation.BWNOT) { if (sort == Sort.DEF) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_NOT_CALL); + org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type); + writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR); } else { if (sort == Sort.INT) { writer.push(-1); @@ -204,15 +215,21 @@ public final class EUnary extends AExpression { throw createError(new IllegalStateException("Illegal tree structure.")); } - writer.math(MethodWriter.XOR, type); + writer.math(MethodWriter.XOR, actual.type); } } else if (operation == Operation.SUB) { if (sort == Sort.DEF) { - writer.invokeStatic(DEF_UTIL_TYPE, DEF_NEG_CALL); + org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type); + writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR); } else { - writer.math(MethodWriter.NEG, type); + writer.math(MethodWriter.NEG, actual.type); } - } else if (operation != Operation.ADD) { + } else if (operation == Operation.ADD) { + if (sort == Sort.DEF) { + org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type); + writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR); + } + } else { throw createError(new IllegalStateException("Illegal tree structure.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java index c80cc8b8ada..38c1ae43907 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -38,7 +38,7 @@ public final class LArrayLength extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if ("length".equals(value)) { if (!load) { throw createError(new IllegalArgumentException("Must read array field [length].")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java index b0816540c5e..91346b7a208 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Sort; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -42,7 +42,7 @@ public final class LBrace extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before == null) { throw createError(new IllegalArgumentException("Illegal array access made without target.")); } @@ -51,18 +51,18 @@ public final class LBrace extends ALink { if (sort == Sort.ARRAY) { index.expected = Definition.INT_TYPE; - index.analyze(variables); - index = index.cast(variables); + index.analyze(locals); + index = index.cast(locals); after = Definition.getType(before.struct, before.dimensions - 1); return this; } else if (sort == Sort.DEF) { - return new LDefArray(location, index).copy(this).analyze(variables); + return new LDefArray(location, index).copy(this).analyze(locals); } else if (Map.class.isAssignableFrom(before.clazz)) { - return new LMapShortcut(location, index).copy(this).analyze(variables); + return new LMapShortcut(location, index).copy(this).analyze(locals); } else if (List.class.isAssignableFrom(before.clazz)) { - return new LListShortcut(location, index).copy(this).analyze(variables); + return new LListShortcut(location, index).copy(this).analyze(locals); } throw createError(new IllegalArgumentException("Illegal array access on type [" + before.name + "].")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java similarity index 79% rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java index 53e66ee68b1..1f6e899b1dd 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java @@ -19,12 +19,12 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.Definition.MethodKey; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Struct; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -32,14 +32,14 @@ import java.util.List; /** * Represents a method call or deferes to a def call. */ -public final class LCall extends ALink { +public final class LCallInvoke extends ALink { final String name; final List arguments; Method method = null; - public LCall(Location location, String name, List arguments) { + public LCallInvoke(Location location, String name, List arguments) { super(location, -1); this.name = name; @@ -47,7 +47,7 @@ public final class LCall extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before == null) { throw createError(new IllegalArgumentException("Illegal call [" + name + "] made without target.")); } else if (before.sort == Sort.ARRAY) { @@ -56,7 +56,7 @@ public final class LCall extends ALink { throw createError(new IllegalArgumentException("Cannot assign a value to a call [" + name + "].")); } - Definition.MethodKey methodKey = new Definition.MethodKey(name, arguments.size()); + MethodKey methodKey = new MethodKey(name, arguments.size()); Struct struct = before.struct; method = statik ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey); @@ -66,8 +66,8 @@ public final class LCall extends ALink { expression.expected = method.arguments.get(argument); expression.internal = true; - expression.analyze(variables); - arguments.set(argument, expression.cast(variables)); + expression.analyze(locals); + arguments.set(argument, expression.cast(locals)); } statement = true; @@ -78,11 +78,11 @@ public final class LCall extends ALink { ALink link = new LDefCall(location, name, arguments); link.copy(this); - return link.analyze(variables); + return link.analyze(locals); } - throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + - "] arguments on type [" + struct.name + "].")); + throw createError(new IllegalArgumentException( + "Unknown call [" + name + "] with [" + arguments.size() + "] arguments on type [" + struct.name + "].")); } @Override @@ -105,10 +105,6 @@ public final class LCall extends ALink { } else { writer.invokeVirtual(method.owner.type, method.method); } - - if (!method.rtn.clazz.equals(method.handle.type().returnType())) { - writer.checkCast(method.rtn.type); - } } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java new file mode 100644 index 00000000000..31085171bf6 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.node; + +import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.MethodWriter; + +import java.util.List; + +import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; + +/** + * Represents a user-defined call. + */ +public class LCallLocal extends ALink { + + final String name; + final List arguments; + + Method method = null; + + public LCallLocal(Location location, String name, List arguments) { + super(location, -1); + + this.name = name; + this.arguments = arguments; + } + + @Override + ALink analyze(Locals locals) { + if (before != null) { + throw createError(new IllegalArgumentException("Illegal call [" + name + "] against an existing target.")); + } else if (store) { + throw createError(new IllegalArgumentException("Cannot assign a value to a call [" + name + "].")); + } + + MethodKey methodKey = new MethodKey(name, arguments.size()); + method = locals.getMethod(methodKey); + + if (method != null) { + for (int argument = 0; argument < arguments.size(); ++argument) { + AExpression expression = arguments.get(argument); + + expression.expected = method.arguments.get(argument); + expression.internal = true; + expression.analyze(locals); + arguments.set(argument, expression.cast(locals)); + } + + statement = true; + after = method.rtn; + + return this; + } + + throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments.")); + } + + @Override + void write(MethodWriter writer) { + // Do nothing. + } + + @Override + void load(MethodWriter writer) { + writer.writeDebugInfo(location); + + for (AExpression argument : arguments) { + argument.write(writer); + } + + writer.invokeStatic(CLASS_TYPE, method.method); + } + + @Override + void store(MethodWriter writer) { + throw createError(new IllegalStateException("Illegal tree structure.")); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java index de4a22a5ad8..d2b4f83a823 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java @@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Cast; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -42,7 +42,7 @@ public final class LCast extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before == null) { throw createError(new IllegalStateException("Illegal cast without a target.")); } else if (store) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java index 5e28cd31b98..5a5333aabe8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Type; import org.elasticsearch.painless.MethodWriter; @@ -42,10 +42,10 @@ final class LDefArray extends ALink implements IDefLink { } @Override - ALink analyze(Variables variables) { - index.analyze(variables); + ALink analyze(Locals locals) { + index.analyze(locals); index.expected = index.actual; - index = index.cast(variables); + index = index.cast(locals); after = Definition.DEF_TYPE; @@ -62,7 +62,7 @@ final class LDefArray extends ALink implements IDefLink { writer.writeDebugInfo(location); String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type); - writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD, 0); + writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD); } @Override @@ -70,6 +70,6 @@ final class LDefArray extends ALink implements IDefLink { writer.writeDebugInfo(location); String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type); - writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE, 0); + writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java index 28c57744860..5301dd2b08d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -46,25 +46,29 @@ final class LDefCall extends ALink implements IDefLink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (arguments.size() > 63) { // technically, the limitation is just methods with > 63 params, containing method references. // this is because we are lazy and use a long as a bitset. we can always change to a "string" if need be. // but NEED NOT BE. nothing with this many parameters is in the whitelist and we do not support varargs. throw new UnsupportedOperationException("methods with > 63 arguments are currently not supported"); } - + recipe = 0; + int totalCaptures = 0; for (int argument = 0; argument < arguments.size(); ++argument) { AExpression expression = arguments.get(argument); if (expression instanceof EFunctionRef) { - recipe |= (1L << argument); // mark argument as deferred reference + recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference + } else if (expression instanceof ECapturingFunctionRef) { + recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference + totalCaptures++; } expression.internal = true; - expression.analyze(variables); + expression.analyze(locals); expression.expected = expression.actual; - arguments.set(argument, expression.cast(variables)); + arguments.set(argument, expression.cast(locals)); } statement = true; @@ -90,6 +94,10 @@ final class LDefCall extends ALink implements IDefLink { for (AExpression argument : arguments) { signature.append(argument.actual.type.getDescriptor()); + if (argument instanceof ECapturingFunctionRef) { + ECapturingFunctionRef capturingRef = (ECapturingFunctionRef) argument; + signature.append(capturingRef.captured.type.type.getDescriptor()); + } argument.write(writer); } @@ -97,7 +105,7 @@ final class LDefCall extends ALink implements IDefLink { // return value signature.append(after.type.getDescriptor()); - writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.METHOD_CALL, recipe); + writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL, recipe); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java index ec9d5bbe9e7..09b48a94f86 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Type; import org.elasticsearch.painless.MethodWriter; @@ -43,7 +43,7 @@ final class LDefField extends ALink implements IDefLink { @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { after = Definition.DEF_TYPE; return this; @@ -59,7 +59,7 @@ final class LDefField extends ALink implements IDefLink { writer.writeDebugInfo(location); String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type); - writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD, 0); + writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD); } @Override @@ -67,6 +67,6 @@ final class LDefField extends ALink implements IDefLink { writer.writeDebugInfo(location); String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type); - writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE, 0); + writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java index 8f34692d666..049dd2d8524 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java @@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Field; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Struct; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -46,7 +46,7 @@ public final class LField extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before == null) { throw createError(new IllegalArgumentException("Illegal field [" + value + "] access made without target.")); } @@ -54,9 +54,9 @@ public final class LField extends ALink { Sort sort = before.sort; if (sort == Sort.ARRAY) { - return new LArrayLength(location, value).copy(this).analyze(variables); + return new LArrayLength(location, value).copy(this).analyze(locals); } else if (sort == Sort.DEF) { - return new LDefField(location, value).copy(this).analyze(variables); + return new LDefField(location, value).copy(this).analyze(locals); } Struct struct = before.struct; @@ -81,17 +81,17 @@ public final class LField extends ALink { Character.toUpperCase(value.charAt(0)) + value.substring(1), 1)); if (shortcut) { - return new LShortcut(location, value).copy(this).analyze(variables); + return new LShortcut(location, value).copy(this).analyze(locals); } else { EConstant index = new EConstant(location, value); - index.analyze(variables); + index.analyze(locals); if (Map.class.isAssignableFrom(before.clazz)) { - return new LMapShortcut(location, index).copy(this).analyze(variables); + return new LMapShortcut(location, index).copy(this).analyze(locals); } if (List.class.isAssignableFrom(before.clazz)) { - return new LListShortcut(location, index).copy(this).analyze(variables); + return new LListShortcut(location, index).copy(this).analyze(locals); } } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java index a8252b40770..7b0feb094e1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java @@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.Sort; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -42,7 +42,7 @@ final class LListShortcut extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { getter = before.struct.methods.get(new Definition.MethodKey("get", 1)); setter = before.struct.methods.get(new Definition.MethodKey("set", 2)); @@ -62,8 +62,8 @@ final class LListShortcut extends ALink { if ((load || store) && (!load || getter != null) && (!store || setter != null)) { index.expected = Definition.INT_TYPE; - index.analyze(variables); - index = index.cast(variables); + index.analyze(locals); + index = index.cast(locals); after = setter != null ? setter.arguments.get(1) : getter.rtn; } else { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java index f31179d135a..5d4c5a9e50a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java @@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.Sort; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -42,7 +42,7 @@ final class LMapShortcut extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { getter = before.struct.methods.get(new Definition.MethodKey("get", 1)); setter = before.struct.methods.get(new Definition.MethodKey("put", 2)); @@ -61,8 +61,8 @@ final class LMapShortcut extends ALink { if ((load || store) && (!load || getter != null) && (!store || setter != null)) { index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0); - index.analyze(variables); - index = index.cast(variables); + index.analyze(locals); + index = index.cast(locals); after = setter != null ? setter.arguments.get(1) : getter.rtn; } else { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java index 75a4aecbe19..15aa01c1d26 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Type; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -43,7 +43,7 @@ public final class LNewArray extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before != null) { throw createError(new IllegalArgumentException("Cannot create a new array with a target already defined.")); } else if (store) { @@ -64,8 +64,8 @@ public final class LNewArray extends ALink { AExpression expression = arguments.get(argument); expression.expected = Definition.INT_TYPE; - expression.analyze(variables); - arguments.set(argument, expression.cast(variables)); + expression.analyze(locals); + arguments.set(argument, expression.cast(locals)); } after = Definition.getType(type.struct, arguments.size()); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java index 2f80b254350..aeb6f64f9db 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java @@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.Struct; import org.elasticsearch.painless.Definition.Type; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; import java.util.List; @@ -47,7 +47,7 @@ public final class LNewObj extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before != null) { throw createError(new IllegalArgumentException("Illegal new call with a target already defined.")); } else if (store) { @@ -79,8 +79,8 @@ public final class LNewObj extends ALink { expression.expected = types[argument]; expression.internal = true; - expression.analyze(variables); - arguments.set(argument, expression.cast(variables)); + expression.analyze(locals); + arguments.set(argument, expression.cast(locals)); } statement = true; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java new file mode 100644 index 00000000000..d91bc3394ee --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.node; + +import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Constant; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.WriterConstants; + +/** + * Represents a regex constant. All regexes are constants. + */ +public final class LRegex extends ALink { + private static final Definition.Type PATTERN_TYPE = Definition.getType("Pattern"); + + private final String pattern; + private Constant constant; + + public LRegex(Location location, String pattern) { + super(location, 1); + this.pattern = pattern; + try { + // Compile the pattern early after parsing so we can throw an error to the user with the location + Pattern.compile(pattern); + } catch (PatternSyntaxException e) { + throw createError(e); + } + } + + @Override + ALink analyze(Locals locals) { + if (before != null) { + throw createError(new IllegalArgumentException("Illegal Regex constant [" + pattern + "].")); + } else if (store) { + throw createError(new IllegalArgumentException("Cannot write to Regex constant [" + pattern + "].")); + } else if (!load) { + throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "].")); + } + + constant = locals.addConstant(location, PATTERN_TYPE, "regexAt$" + location.getOffset(), this::initializeConstant); + after = PATTERN_TYPE; + + return this; + } + + @Override + void write(MethodWriter writer) { + // Do nothing. + } + + @Override + void load(MethodWriter writer) { + writer.writeDebugInfo(location); + writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, PATTERN_TYPE.type); + } + + @Override + void store(MethodWriter writer) { + throw createError(new IllegalStateException("Illegal tree structure.")); + } + + private void initializeConstant(MethodWriter writer) { + writer.push(pattern); + writer.invokeStatic(PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java index a91970fa577..73cbc201db4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java @@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Method; import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Struct; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -44,7 +44,7 @@ final class LShortcut extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { Struct struct = before.struct; getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java index 98774513188..6f04e9a22c4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java @@ -22,7 +22,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; /** * Represents a static type target. @@ -38,7 +38,7 @@ public final class LStatic extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before != null) { throw createError(new IllegalArgumentException("Illegal static type [" + type + "] after target already defined.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java index 6afe62d02ba..41eb027e3ab 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -36,7 +36,7 @@ public final class LString extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before != null) { throw createError(new IllegalArgumentException("Illegal String constant [" + string + "].")); } else if (store) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java index 8fe6f17b0b5..64dc852117d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java @@ -21,8 +21,8 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.Variables; -import org.elasticsearch.painless.Variables.Variable; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Variable; import org.objectweb.asm.Opcodes; /** @@ -41,12 +41,12 @@ public final class LVariable extends ALink { } @Override - ALink analyze(Variables variables) { + ALink analyze(Locals locals) { if (before != null) { throw createError(new IllegalArgumentException("Illegal variable [" + name + "] access with target already defined.")); } - Variable variable = variables.getVariable(location, name); + Variable variable = locals.getVariable(location, name); if (store && variable.readonly) { throw createError(new IllegalArgumentException("Variable [" + variable.name + "] is read-only.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java index 4dbbd80de54..5fec362ec17 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -40,7 +40,7 @@ public final class SBlock extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (statements == null || statements.isEmpty()) { throw createError(new IllegalArgumentException("A block must contain at least one statement.")); } @@ -58,7 +58,7 @@ public final class SBlock extends AStatement { statement.lastSource = lastSource && statement == last; statement.lastLoop = (beginLoop || lastLoop) && statement == last; - statement.analyze(variables); + statement.analyze(locals); methodEscape = statement.methodEscape; loopEscape = statement.loopEscape; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java index ca72dd0b55b..d4f8bfff4e4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -33,7 +33,7 @@ public final class SBreak extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (!inLoop) { throw createError(new IllegalArgumentException("Break statement outside of a loop.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java index 8bcaf9d22cf..42de28e48f5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java @@ -22,8 +22,8 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Type; -import org.elasticsearch.painless.Variables; -import org.elasticsearch.painless.Variables.Variable; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Variable; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; import org.elasticsearch.painless.MethodWriter; @@ -52,7 +52,7 @@ public final class SCatch extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { final Type type; try { @@ -65,14 +65,14 @@ public final class SCatch extends AStatement { throw createError(new ClassCastException("Not an exception type [" + this.type + "].")); } - variable = variables.addVariable(location, type, name, true, false); + variable = locals.addVariable(location, type, name, true, false); if (block != null) { block.lastSource = lastSource; block.inLoop = inLoop; block.lastLoop = lastLoop; - block.analyze(variables); + block.analyze(locals); methodEscape = block.methodEscape; loopEscape = block.loopEscape; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java index ef80766bdd1..802cf90087b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -33,7 +33,7 @@ public final class SContinue extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (!inLoop) { throw createError(new IllegalArgumentException("Continue statement outside of a loop.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java index 1ff187afe29..5365e72888a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -40,9 +40,9 @@ public final class SDeclBlock extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { for (SDeclaration declaration : declarations) { - declaration.analyze(variables); + declaration.analyze(locals); } statementCount = declarations.size(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java index 5f039184cb6..a62a287df9c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java @@ -22,8 +22,8 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Type; -import org.elasticsearch.painless.Variables; -import org.elasticsearch.painless.Variables.Variable; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Variable; import org.objectweb.asm.Opcodes; import org.elasticsearch.painless.MethodWriter; @@ -47,7 +47,7 @@ public final class SDeclaration extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { final Type type; try { @@ -58,11 +58,11 @@ public final class SDeclaration extends AStatement { if (expression != null) { expression.expected = type; - expression.analyze(variables); - expression = expression.cast(variables); + expression.analyze(locals); + expression = expression.cast(locals); } - variable = variables.addVariable(location, type, name, false, false); + variable = locals.addVariable(location, type, name, false, false); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java index 4989fb77a79..b25d903dc41 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -30,21 +30,19 @@ import org.elasticsearch.painless.MethodWriter; */ public final class SDo extends AStatement { - final int maxLoopCounter; final SBlock block; AExpression condition; - public SDo(Location location, int maxLoopCounter, SBlock block, AExpression condition) { + public SDo(Location location, SBlock block, AExpression condition) { super(location); this.condition = condition; this.block = block; - this.maxLoopCounter = maxLoopCounter; } @Override - void analyze(Variables variables) { - variables.incrementScope(); + void analyze(Locals locals) { + locals.incrementScope(); if (block == null) { throw createError(new IllegalArgumentException("Extraneous do while loop.")); @@ -53,15 +51,15 @@ public final class SDo extends AStatement { block.beginLoop = true; block.inLoop = true; - block.analyze(variables); + block.analyze(locals); if (block.loopEscape && !block.anyContinue) { throw createError(new IllegalArgumentException("Extraneous do while loop.")); } condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); if (condition.constant != null) { final boolean continuous = (boolean)condition.constant; @@ -78,11 +76,11 @@ public final class SDo extends AStatement { statementCount = 1; - if (maxLoopCounter > 0) { - loopCounterSlot = variables.getVariable(location, "#loop").slot; + if (locals.getMaxLoopCounter() > 0) { + loopCounterSlot = locals.getVariable(location, "#loop").slot; } - variables.decrementScope(); + locals.decrementScope(); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java index 9a6fe798957..48a3f6e3eae 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java @@ -29,8 +29,8 @@ import org.elasticsearch.painless.Definition.Sort; import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.Variables; -import org.elasticsearch.painless.Variables.Variable; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Variable; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; @@ -44,7 +44,6 @@ import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE; */ public class SEach extends AStatement { - final int maxLoopCounter; final String type; final String name; AExpression expression; @@ -63,10 +62,9 @@ public class SEach extends AStatement { Variable iterator = null; Method method = null; - public SEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) { + public SEach(Location location, String type, String name, AExpression expression, SBlock block) { super(location); - this.maxLoopCounter = maxLoopCounter; this.type = type; this.name = name; this.expression = expression; @@ -74,10 +72,10 @@ public class SEach extends AStatement { } @Override - void analyze(Variables variables) { - expression.analyze(variables); + void analyze(Locals locals) { + expression.analyze(locals); expression.expected = expression.actual; - expression = expression.cast(variables); + expression = expression.cast(locals); final Type type; @@ -87,25 +85,25 @@ public class SEach extends AStatement { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } - variables.incrementScope(); + locals.incrementScope(); - variable = variables.addVariable(location, type, name, true, false); + variable = locals.addVariable(location, type, name, true, false); if (expression.actual.sort == Sort.ARRAY) { - analyzeArray(variables, type); + analyzeArray(locals, type); } else if (expression.actual.sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) { - analyzeIterable(variables, type); + analyzeIterable(locals, type); } else { - throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "].")); + throw createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "].")); } if (block == null) { - throw location.createError(new IllegalArgumentException("Extraneous for each loop.")); + throw createError(new IllegalArgumentException("Extraneous for each loop.")); } block.beginLoop = true; block.inLoop = true; - block.analyze(variables); + block.analyze(locals); block.statementCount = Math.max(1, block.statementCount); if (block.loopEscape && !block.anyContinue) { @@ -114,14 +112,14 @@ public class SEach extends AStatement { statementCount = 1; - if (maxLoopCounter > 0) { - loopCounterSlot = variables.getVariable(location, "#loop").slot; + if (locals.getMaxLoopCounter() > 0) { + loopCounterSlot = locals.getVariable(location, "#loop").slot; } - variables.decrementScope(); + locals.decrementScope(); } - void analyzeArray(Variables variables, Type type) { + void analyzeArray(Locals variables, Type type) { // We must store the array and index as variables for securing slots on the stack, and // also add the location offset to make the names unique in case of nested for each loops. array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false); @@ -130,7 +128,7 @@ public class SEach extends AStatement { cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true); } - void analyzeIterable(Variables variables, Type type) { + void analyzeIterable(Locals variables, Type type) { // We must store the iterator as a variable for securing a slot on the stack, and // also add the location offset to make the name unique in case of nested for each loops. iterator = variables.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true, false); @@ -141,7 +139,7 @@ public class SEach extends AStatement { method = expression.actual.struct.methods.get(new MethodKey("iterator", 0)); if (method == null) { - throw location.createError(new IllegalArgumentException( + throw createError(new IllegalArgumentException( "Unable to create iterator for the type [" + expression.actual.name + "].")); } } @@ -158,7 +156,7 @@ public class SEach extends AStatement { } else if (iterator != null) { writeIterable(writer); } else { - throw location.createError(new IllegalStateException("Illegal tree structure.")); + throw createError(new IllegalStateException("Illegal tree structure.")); } } @@ -197,7 +195,7 @@ public class SEach extends AStatement { if (method == null) { Type itr = Definition.getType("Iterator"); String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type); - writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR, 0); + writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ITERATOR); } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) { writer.invokeInterface(method.owner.type, method.method); } else { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java index 1bea07d5599..8817db95be6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java @@ -19,10 +19,10 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.Definition.Type; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Definition.Sort; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -39,19 +39,22 @@ public final class SExpression extends AStatement { } @Override - void analyze(Variables variables) { - expression.read = lastSource; - expression.analyze(variables); + void analyze(Locals locals) { + Type rtnType = locals.getReturnType(); + boolean isVoid = rtnType.sort == Sort.VOID; + + expression.read = lastSource && !isVoid; + expression.analyze(locals); if (!lastSource && !expression.statement) { throw createError(new IllegalArgumentException("Not a statement.")); } - final boolean rtn = lastSource && expression.actual.sort != Sort.VOID; + boolean rtn = lastSource && !isVoid && expression.actual.sort != Sort.VOID; - expression.expected = rtn ? Definition.OBJECT_TYPE : expression.actual; + expression.expected = rtn ? rtnType : expression.actual; expression.internal = rtn; - expression = expression.cast(variables); + expression = expression.cast(locals); methodEscape = rtn; loopEscape = rtn; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java index 04475a91a1a..cc9dee20138 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -30,17 +30,14 @@ import org.elasticsearch.painless.MethodWriter; */ public final class SFor extends AStatement { - final int maxLoopCounter; ANode initializer; AExpression condition; AExpression afterthought; final SBlock block; - public SFor(Location location, int maxLoopCounter, - ANode initializer, AExpression condition, AExpression afterthought, SBlock block) { + public SFor(Location location, ANode initializer, AExpression condition, AExpression afterthought, SBlock block) { super(location); - this.maxLoopCounter = maxLoopCounter; this.initializer = initializer; this.condition = condition; this.afterthought = afterthought; @@ -48,19 +45,19 @@ public final class SFor extends AStatement { } @Override - void analyze(Variables variables) { - variables.incrementScope(); + void analyze(Locals locals) { + locals.incrementScope(); boolean continuous = false; if (initializer != null) { if (initializer instanceof AStatement) { - ((AStatement)initializer).analyze(variables); + ((AStatement)initializer).analyze(locals); } else if (initializer instanceof AExpression) { AExpression initializer = (AExpression)this.initializer; initializer.read = false; - initializer.analyze(variables); + initializer.analyze(locals); if (!initializer.statement) { throw createError(new IllegalArgumentException("Not a statement.")); @@ -72,8 +69,8 @@ public final class SFor extends AStatement { if (condition != null) { condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); if (condition.constant != null) { continuous = (boolean)condition.constant; @@ -92,7 +89,7 @@ public final class SFor extends AStatement { if (afterthought != null) { afterthought.read = false; - afterthought.analyze(variables); + afterthought.analyze(locals); if (!afterthought.statement) { throw createError(new IllegalArgumentException("Not a statement.")); @@ -103,7 +100,7 @@ public final class SFor extends AStatement { block.beginLoop = true; block.inLoop = true; - block.analyze(variables); + block.analyze(locals); if (block.loopEscape && !block.anyContinue) { throw createError(new IllegalArgumentException("Extraneous for loop.")); @@ -119,11 +116,11 @@ public final class SFor extends AStatement { statementCount = 1; - if (maxLoopCounter > 0) { - loopCounterSlot = variables.getVariable(location, "#loop").slot; + if (locals.getMaxLoopCounter() > 0) { + loopCounterSlot = locals.getVariable(location, "#loop").slot; } - variables.decrementScope(); + locals.decrementScope(); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java new file mode 100644 index 00000000000..0cdde13fce7 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.node; + +import org.elasticsearch.painless.Def; +import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.Definition.Sort; +import org.elasticsearch.painless.Definition.Type; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Parameter; +import org.elasticsearch.painless.Locals.FunctionReserved; +import org.elasticsearch.painless.Locals.Variable; +import org.elasticsearch.painless.Location; +import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.WriterConstants; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.MethodType; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; + +/** + * Represents a user-defined function. + */ +public class SFunction extends AStatement { + final FunctionReserved reserved; + final String rtnTypeStr; + final String name; + final List paramTypeStrs; + final List paramNameStrs; + final List statements; + final boolean synthetic; + + Type rtnType = null; + List parameters = new ArrayList<>(); + Method method = null; + + Locals locals = null; + + public SFunction(FunctionReserved reserved, Location location, + String rtnType, String name, List paramTypes, + List paramNames, List statements, boolean synthetic) { + super(location); + + this.reserved = reserved; + this.rtnTypeStr = rtnType; + this.name = name; + this.paramTypeStrs = Collections.unmodifiableList(paramTypes); + this.paramNameStrs = Collections.unmodifiableList(paramNames); + this.statements = Collections.unmodifiableList(statements); + this.synthetic = synthetic; + } + + void generate() { + try { + rtnType = Definition.getType(rtnTypeStr); + } catch (IllegalArgumentException exception) { + throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "].")); + } + + if (paramTypeStrs.size() != paramNameStrs.size()) { + throw createError(new IllegalStateException("Illegal tree structure.")); + } + + Class[] paramClasses = new Class[this.paramTypeStrs.size()]; + List paramTypes = new ArrayList<>(); + + for (int param = 0; param < this.paramTypeStrs.size(); ++param) { + try { + Type paramType = Definition.getType(this.paramTypeStrs.get(param)); + + paramClasses[param] = paramType.clazz; + paramTypes.add(paramType); + parameters.add(new Parameter(location, paramNameStrs.get(param), paramType)); + } catch (IllegalArgumentException exception) { + throw createError(new IllegalArgumentException( + "Illegal parameter type [" + this.paramTypeStrs.get(param) + "] for function [" + name + "].")); + } + } + + org.objectweb.asm.commons.Method method = + new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtnType.clazz, paramClasses).toMethodDescriptorString()); + this.method = new Method(name, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null); + } + + @Override + void analyze(Locals locals) { + if (statements == null || statements.isEmpty()) { + throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "].")); + } + + this.locals = new Locals(reserved, locals, rtnType, parameters); + locals = this.locals; + + locals.incrementScope(); + + AStatement last = statements.get(statements.size() - 1); + + for (AStatement statement : statements) { + // Note that we do not need to check after the last statement because + // there is no statement that can be unreachable after the last. + if (allEscape) { + throw createError(new IllegalArgumentException("Unreachable statement.")); + } + + statement.lastSource = statement == last; + + statement.analyze(locals); + + methodEscape = statement.methodEscape; + allEscape = statement.allEscape; + } + + if (!methodEscape && rtnType.sort != Sort.VOID) { + throw createError(new IllegalArgumentException("Not all paths provide a return value for method [" + name + "].")); + } + + locals.decrementScope(); + + String staticHandleFieldName = Def.getUserFunctionHandleFieldName(name, parameters.size()); + locals.addConstant(location, WriterConstants.METHOD_HANDLE_TYPE, staticHandleFieldName, this::initializeConstant); + } + + /** Writes the function to given ClassWriter. */ + void write (ClassWriter writer, BitSet statements) { + int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC; + if (synthetic) { + access |= Opcodes.ACC_SYNTHETIC; + } + final MethodWriter function = new MethodWriter(access, method.method, writer, statements); + write(function); + function.endMethod(); + } + + @Override + void write(MethodWriter function) { + if (reserved.getMaxLoopCounter() > 0) { + // if there is infinite loop protection, we do this once: + // int #loop = settings.getMaxLoopCounter() + + Variable loop = locals.getVariable(null, FunctionReserved.LOOP); + + function.push(reserved.getMaxLoopCounter()); + function.visitVarInsn(Opcodes.ISTORE, loop.slot); + } + + for (AStatement statement : statements) { + statement.write(function); + } + + if (!methodEscape) { + if (rtnType.sort == Sort.VOID) { + function.returnValue(); + } else { + throw createError(new IllegalStateException("Illegal tree structure.")); + } + } + } + + private void initializeConstant(MethodWriter writer) { + final Handle handle = new Handle(Opcodes.H_INVOKESTATIC, + CLASS_TYPE.getInternalName(), + name, + method.method.getDescriptor(), + false); + writer.push(handle); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java index 954ddac9c6a..a46075af9a1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -41,10 +41,10 @@ public final class SIf extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); if (condition.constant != null) { throw createError(new IllegalArgumentException("Extraneous if statement.")); @@ -58,9 +58,9 @@ public final class SIf extends AStatement { ifblock.inLoop = inLoop; ifblock.lastLoop = lastLoop; - variables.incrementScope(); - ifblock.analyze(variables); - variables.decrementScope(); + locals.incrementScope(); + ifblock.analyze(locals); + locals.decrementScope(); anyContinue = ifblock.anyContinue; anyBreak = ifblock.anyBreak; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java index 1d801267054..22cbfe25614 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -43,10 +43,10 @@ public final class SIfElse extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); if (condition.constant != null) { throw createError(new IllegalArgumentException("Extraneous if statement.")); @@ -60,9 +60,9 @@ public final class SIfElse extends AStatement { ifblock.inLoop = inLoop; ifblock.lastLoop = lastLoop; - variables.incrementScope(); - ifblock.analyze(variables); - variables.decrementScope(); + locals.incrementScope(); + ifblock.analyze(locals); + locals.decrementScope(); anyContinue = ifblock.anyContinue; anyBreak = ifblock.anyBreak; @@ -76,9 +76,9 @@ public final class SIfElse extends AStatement { elseblock.inLoop = inLoop; elseblock.lastLoop = lastLoop; - variables.incrementScope(); - elseblock.analyze(variables); - variables.decrementScope(); + locals.incrementScope(); + elseblock.analyze(locals); + locals.decrementScope(); methodEscape = ifblock.methodEscape && elseblock.methodEscape; loopEscape = ifblock.loopEscape && elseblock.loopEscape; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java index bd9707eb6a4..e789f3372e3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java @@ -19,9 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -38,11 +37,11 @@ public final class SReturn extends AStatement { } @Override - void analyze(Variables variables) { - expression.expected = Definition.OBJECT_TYPE; + void analyze(Locals locals) { + expression.expected = locals.getReturnType(); expression.internal = true; - expression.analyze(variables); - expression = expression.cast(variables); + expression.analyze(locals); + expression = expression.cast(locals); methodEscape = true; loopEscape = true; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java index 22fd2e3e28c..aeda0387297 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java @@ -19,34 +19,87 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; -import org.objectweb.asm.Opcodes; +import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.Executable; +import org.elasticsearch.painless.Locals; +import org.elasticsearch.painless.Locals.Constant; +import org.elasticsearch.painless.Locals.ExecuteReserved; +import org.elasticsearch.painless.Locals.Variable; +import org.elasticsearch.painless.WriterConstants; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Opcodes; + +import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE; +import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE; +import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR; +import static org.elasticsearch.painless.WriterConstants.EXECUTE; +import static org.elasticsearch.painless.WriterConstants.MAP_GET; +import static org.elasticsearch.painless.WriterConstants.MAP_TYPE; /** * The root of all Painless trees. Contains a series of statements. */ public final class SSource extends AStatement { + final String name; + final String source; + final ExecuteReserved reserved; + final List functions; final List statements; - public SSource(Location location, List statements) { + private Locals locals; + private BitSet expressions; + private byte[] bytes; + + public SSource(String name, String source, ExecuteReserved reserved, Location location, + List functions, List statements) { super(location); + this.name = name; + this.source = source; + this.reserved = reserved; + this.functions = Collections.unmodifiableList(functions); this.statements = Collections.unmodifiableList(statements); } + public void analyze() { + Map methods = new HashMap<>(); + + for (SFunction function : functions) { + function.generate(); + + MethodKey key = new MethodKey(function.name, function.parameters.size()); + + if (methods.put(key, function.method) != null) { + throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "].")); + } + } + + locals = new Locals(reserved, methods); + analyze(locals); + } + @Override - public void analyze(Variables variables) { + void analyze(Locals locals) { + for (SFunction function : functions) { + function.analyze(locals); + } + if (statements == null || statements.isEmpty()) { throw createError(new IllegalArgumentException("Cannot generate an empty script.")); } - variables.incrementScope(); + locals.incrementScope(); AStatement last = statements.get(statements.size() - 1); @@ -59,17 +112,115 @@ public final class SSource extends AStatement { statement.lastSource = statement == last; - statement.analyze(variables); + statement.analyze(locals); methodEscape = statement.methodEscape; allEscape = statement.allEscape; } - variables.decrementScope(); + locals.decrementScope(); } + public void write() { + // Create the ClassWriter. + + int classFrames = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS; + int classVersion = Opcodes.V1_8; + int classAccess = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL; + String classBase = BASE_CLASS_TYPE.getInternalName(); + String className = CLASS_TYPE.getInternalName(); + String classInterfaces[] = reserved.usesScore() ? new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null; + + ClassWriter writer = new ClassWriter(classFrames); + writer.visit(classVersion, classAccess, className, null, classBase, classInterfaces); + writer.visitSource(Location.computeSourceName(name, source), null); + + expressions = new BitSet(source.length()); + + // Write the constructor: + MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, writer, expressions); + constructor.loadThis(); + constructor.loadArgs(); + constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR); + constructor.returnValue(); + constructor.endMethod(); + + // Write the execute method: + MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions); + write(execute); + execute.endMethod(); + + // Write all functions: + for (SFunction function : functions) { + function.write(writer, expressions); + } + + // Write the constants + if (false == locals.getConstants().isEmpty()) { + // Fields + for (Constant constant : locals.getConstants()) { + writer.visitField( + Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, + constant.name, + constant.type.getDescriptor(), + null, + null).visitEnd(); + } + + // Initialize the constants in a static initializer + final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, + WriterConstants.CLINIT, writer, expressions); + for (Constant constant : locals.getConstants()) { + constant.initializer.accept(clinit); + clinit.putStatic(CLASS_TYPE, constant.name, constant.type); + } + clinit.returnValue(); + clinit.endMethod(); + } + + // End writing the class and store the generated bytes. + + writer.visitEnd(); + bytes = writer.toByteArray(); + } + @Override - public void write(MethodWriter writer) { + void write(MethodWriter writer) { + if (reserved.usesScore()) { + // if the _score value is used, we do this once: + // final double _score = scorer.score(); + Variable scorer = locals.getVariable(null, ExecuteReserved.SCORER); + Variable score = locals.getVariable(null, ExecuteReserved.SCORE); + + writer.visitVarInsn(Opcodes.ALOAD, scorer.slot); + writer.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE); + writer.visitInsn(Opcodes.F2D); + writer.visitVarInsn(Opcodes.DSTORE, score.slot); + } + + if (reserved.usesCtx()) { + // if the _ctx value is used, we do this once: + // final Map ctx = input.get("ctx"); + + Variable input = locals.getVariable(null, ExecuteReserved.PARAMS); + Variable ctx = locals.getVariable(null, ExecuteReserved.CTX); + + writer.visitVarInsn(Opcodes.ALOAD, input.slot); + writer.push(ExecuteReserved.CTX); + writer.invokeInterface(MAP_TYPE, MAP_GET); + writer.visitVarInsn(Opcodes.ASTORE, ctx.slot); + } + + if (reserved.getMaxLoopCounter() > 0) { + // if there is infinite loop protection, we do this once: + // int #loop = settings.getMaxLoopCounter() + + Variable loop = locals.getVariable(null, ExecuteReserved.LOOP); + + writer.push(reserved.getMaxLoopCounter()); + writer.visitVarInsn(Opcodes.ISTORE, loop.slot); + } + for (AStatement statement : statements) { statement.write(writer); } @@ -79,4 +230,12 @@ public final class SSource extends AStatement { writer.returnValue(); } } + + public BitSet getExpressions() { + return expressions; + } + + public byte[] getBytes() { + return bytes; + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java index af9d7a65990..db04d622839 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.MethodWriter; /** @@ -38,10 +38,10 @@ public final class SThrow extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { expression.expected = Definition.EXCEPTION_TYPE; - expression.analyze(variables); - expression = expression.cast(variables); + expression.analyze(locals); + expression = expression.cast(locals); methodEscape = true; loopEscape = true; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java index c24c8273dba..42fffc759ce 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -43,7 +43,7 @@ public final class STry extends AStatement { } @Override - void analyze(Variables variables) { + void analyze(Locals locals) { if (block == null) { throw createError(new IllegalArgumentException("Extraneous try statement.")); } @@ -52,9 +52,9 @@ public final class STry extends AStatement { block.inLoop = inLoop; block.lastLoop = lastLoop; - variables.incrementScope(); - block.analyze(variables); - variables.decrementScope(); + locals.incrementScope(); + block.analyze(locals); + locals.decrementScope(); methodEscape = block.methodEscape; loopEscape = block.loopEscape; @@ -69,9 +69,9 @@ public final class STry extends AStatement { catc.inLoop = inLoop; catc.lastLoop = lastLoop; - variables.incrementScope(); - catc.analyze(variables); - variables.decrementScope(); + locals.incrementScope(); + catc.analyze(locals); + locals.decrementScope(); methodEscape &= catc.methodEscape; loopEscape &= catc.loopEscape; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java index 59c1bb75ee8..20478a55aa0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java @@ -21,7 +21,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.Definition; import org.elasticsearch.painless.Location; -import org.elasticsearch.painless.Variables; +import org.elasticsearch.painless.Locals; import org.objectweb.asm.Label; import org.elasticsearch.painless.MethodWriter; @@ -30,25 +30,23 @@ import org.elasticsearch.painless.MethodWriter; */ public final class SWhile extends AStatement { - final int maxLoopCounter; AExpression condition; final SBlock block; - public SWhile(Location location, int maxLoopCounter, AExpression condition, SBlock block) { + public SWhile(Location location, AExpression condition, SBlock block) { super(location); - this.maxLoopCounter = maxLoopCounter; this.condition = condition; this.block = block; } @Override - void analyze(Variables variables) { - variables.incrementScope(); + void analyze(Locals locals) { + locals.incrementScope(); condition.expected = Definition.BOOLEAN_TYPE; - condition.analyze(variables); - condition = condition.cast(variables); + condition.analyze(locals); + condition = condition.cast(locals); boolean continuous = false; @@ -68,7 +66,7 @@ public final class SWhile extends AStatement { block.beginLoop = true; block.inLoop = true; - block.analyze(variables); + block.analyze(locals); if (block.loopEscape && !block.anyContinue) { throw createError(new IllegalArgumentException("Extraneous while loop.")); @@ -84,11 +82,11 @@ public final class SWhile extends AStatement { statementCount = 1; - if (maxLoopCounter > 0) { - loopCounterSlot = variables.getVariable(location, "#loop").slot; + if (locals.getMaxLoopCounter() > 0) { + loopCounterSlot = locals.getVariable(location, "#loop").slot; } - variables.decrementScope(); + locals.decrementScope(); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java index 71cde33e979..d98e5f68bcf 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java @@ -35,6 +35,7 @@ * {@link org.elasticsearch.painless.node.EBinary} - Represents a binary math expression. * {@link org.elasticsearch.painless.node.EBool} - Represents a boolean expression. * {@link org.elasticsearch.painless.node.EBoolean} - Represents a boolean constant. + * {@link org.elasticsearch.painless.node.ECapturingFunctionRef} - Represents a function reference (capturing). * {@link org.elasticsearch.painless.node.ECast} - Represents an implicit cast in most cases. (Internal only.) * {@link org.elasticsearch.painless.node.EChain} - Represents the entirety of a variable/method chain for read/write operations. * {@link org.elasticsearch.painless.node.EComp} - Represents a comparison expression. @@ -42,14 +43,15 @@ * {@link org.elasticsearch.painless.node.EConstant} - Represents a constant. (Internal only.) * {@link org.elasticsearch.painless.node.EDecimal} - Represents a decimal constant. * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast. - * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference. + * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference (non-capturing). * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant. * {@link org.elasticsearch.painless.node.ENumeric} - Represents a non-decimal numeric constant. * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression. * {@link org.elasticsearch.painless.node.IDefLink} - A marker interface for all LDef* (link) nodes. * {@link org.elasticsearch.painless.node.LArrayLength} - Represents an array length field load. * {@link org.elasticsearch.painless.node.LBrace} - Represents an array load/store or defers to possible shortcuts. - * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or defers to a def call. + * {@link org.elasticsearch.painless.node.LCallInvoke} - Represents a method call or defers to a def call. + * {@link org.elasticsearch.painless.node.LCallLocal} - Represents a user-defined call. * {@link org.elasticsearch.painless.node.LCast} - Represents a cast made in a variable/method chain. * {@link org.elasticsearch.painless.node.LDefArray} - Represents an array load/store or shortcut on a def type. (Internal only.) * {@link org.elasticsearch.painless.node.LDefCall} - Represents a method call made on a def type. (Internal only.) @@ -73,6 +75,7 @@ * {@link org.elasticsearch.painless.node.SEach} - Represents a for each loop shortcut for iterables. * {@link org.elasticsearch.painless.node.SExpression} - Represents the top-level node for an expression as a statement. * {@link org.elasticsearch.painless.node.SFor} - Represents a for loop. + * {@link org.elasticsearch.painless.node.SFunction} - Represents a user-defined function. * {@link org.elasticsearch.painless.node.SIf} - Represents an if block. * {@link org.elasticsearch.painless.node.SIfElse} - Represents an if/else block. * {@link org.elasticsearch.painless.node.SReturn} - Represents a return statement. diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt new file mode 100644 index 00000000000..b66793dcf5a --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt @@ -0,0 +1,33 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Painless definition file. This defines the hierarchy of classes, +# what methods and fields they have, etc. +# + +class Pattern -> java.util.regex.Pattern extends Object { +# Pattern compile(String) Intentionally not included. We don't want dynamic patterns because they allow regexes to be generated per time +# the script is run which is super slow. LRegex generates code that calls this method but it skips these checks. + Matcher matcher(CharSequence) +} + +class Matcher -> java.util.regex.Matcher extends Object { + boolean matches() +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt index 7fff9ac0aa4..308f2661ebc 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt @@ -125,4 +125,5 @@ class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.Featu void setY(int) boolean overloadedStatic() boolean overloadedStatic(boolean) + Object twoFunctionsOfX(Function,Function) } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java index 2a8195ca66d..c41ec15bd35 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java @@ -22,6 +22,10 @@ package org.elasticsearch.painless; /** Tests for addition operator across all types */ //TODO: NaN/Inf/overflow/... public class AdditionTests extends ScriptTestCase { + + public void testBasics() throws Exception { + assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;")); + } public void testInt() throws Exception { assertEquals(1+1, exec("int x = 1; int y = 1; return x+y;")); @@ -191,4 +195,178 @@ public class AdditionTests extends ScriptTestCase { assertEquals(1.0+0.0, exec("return 1.0+0.0;")); assertEquals(0.0+0.0, exec("return 0.0+0.0;")); } + + public void testDef() { + assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (byte)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (byte)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x + y")); + assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y")); + assertEquals(2, exec("def x = (char)1; def y = (short)1; return x + y")); + assertEquals(2, exec("def x = (int)1; def y = (short)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (short)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (short)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x + y")); + assertEquals(2, exec("def x = (short)1; def y = (char)1; return x + y")); + assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y")); + assertEquals(2, exec("def x = (int)1; def y = (char)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (char)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (char)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x + y")); + assertEquals(2, exec("def x = (short)1; def y = (int)1; return x + y")); + assertEquals(2, exec("def x = (char)1; def y = (int)1; return x + y")); + assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (int)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (int)1; return x + y")); + + assertEquals(2L, exec("def x = (byte)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (short)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (char)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (int)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (long)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (long)1; return x + y")); + + assertEquals(2F, exec("def x = (byte)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (short)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (char)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (int)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (long)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (float)1; return x + y")); + + assertEquals(2D, exec("def x = (byte)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (short)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (char)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (int)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y")); + } + + public void testDefTypedLHS() { + assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x + y")); + assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x + y")); + assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (byte)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (byte)1; return x + y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x + y")); + assertEquals(2, exec("short x = (short)1; def y = (short)1; return x + y")); + assertEquals(2, exec("char x = (char)1; def y = (short)1; return x + y")); + assertEquals(2, exec("int x = (int)1; def y = (short)1; return x + y")); + assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (short)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (short)1; return x + y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x + y")); + assertEquals(2, exec("short x = (short)1; def y = (char)1; return x + y")); + assertEquals(2, exec("char x = (char)1; def y = (char)1; return x + y")); + assertEquals(2, exec("int x = (int)1; def y = (char)1; return x + y")); + assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (char)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (char)1; return x + y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x + y")); + assertEquals(2, exec("short x = (short)1; def y = (int)1; return x + y")); + assertEquals(2, exec("char x = (char)1; def y = (int)1; return x + y")); + assertEquals(2, exec("int x = (int)1; def y = (int)1; return x + y")); + assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (int)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (int)1; return x + y")); + + assertEquals(2L, exec("byte x = (byte)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("short x = (short)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("char x = (char)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("int x = (int)1; def y = (long)1; return x + y")); + assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (long)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (long)1; return x + y")); + + assertEquals(2F, exec("byte x = (byte)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("short x = (short)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("char x = (char)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("int x = (int)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("long x = (long)1; def y = (float)1; return x + y")); + assertEquals(2F, exec("float x = (float)1; def y = (float)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (float)1; return x + y")); + + assertEquals(2D, exec("byte x = (byte)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("short x = (short)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("char x = (char)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("int x = (int)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("long x = (long)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y")); + assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y")); + } + + public void testDefTypedRHS() { + assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x + y")); + assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; byte y = (byte)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; byte y = (byte)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x + y")); + assertEquals(2, exec("def x = (short)1; short y = (short)1; return x + y")); + assertEquals(2, exec("def x = (char)1; short y = (short)1; return x + y")); + assertEquals(2, exec("def x = (int)1; short y = (short)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; short y = (short)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; short y = (short)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x + y")); + assertEquals(2, exec("def x = (short)1; char y = (char)1; return x + y")); + assertEquals(2, exec("def x = (char)1; char y = (char)1; return x + y")); + assertEquals(2, exec("def x = (int)1; char y = (char)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; char y = (char)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; char y = (char)1; return x + y")); + + assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x + y")); + assertEquals(2, exec("def x = (short)1; int y = (int)1; return x + y")); + assertEquals(2, exec("def x = (char)1; int y = (int)1; return x + y")); + assertEquals(2, exec("def x = (int)1; int y = (int)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; int y = (int)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; int y = (int)1; return x + y")); + + assertEquals(2L, exec("def x = (byte)1; long y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (short)1; long y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (char)1; long y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (int)1; long y = (long)1; return x + y")); + assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; long y = (long)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; long y = (long)1; return x + y")); + + assertEquals(2F, exec("def x = (byte)1; float y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (short)1; float y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (char)1; float y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (int)1; float y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (long)1; float y = (float)1; return x + y")); + assertEquals(2F, exec("def x = (float)1; float y = (float)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; float y = (float)1; return x + y")); + + assertEquals(2D, exec("def x = (byte)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (short)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (char)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (int)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (long)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y")); + assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java index 2c86250da83..42cf3611219 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java @@ -21,6 +21,13 @@ package org.elasticsearch.painless; /** Tests for and operator across all types */ public class AndTests extends ScriptTestCase { + + public void testBasics() throws Exception { + assertEquals(5 & 3, exec("return 5 & 3;")); + assertEquals(5 & 3L, exec("return 5 & 3L;")); + assertEquals(5L & 3, exec("return 5L & 3;")); + assertEquals(1L, exec("int x = 5; long y = 3; return x & y;")); + } public void testInt() throws Exception { assertEquals(5 & 12, exec("int x = 5; int y = 12; return x & y;")); @@ -45,4 +52,160 @@ public class AndTests extends ScriptTestCase { assertEquals(5L & -12L, exec("return 5L & -12L;")); assertEquals(7L & 15L & 3L, exec("return 7L & 15L & 3L;")); } + + public void testIllegal() throws Exception { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; int y = 1; return x & y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; int y = 1; return x & y"); + }); + } + + public void testDef() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; def y = (byte)1; return x & y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; def y = (byte)1; return x & y"); + }); + assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y")); + assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y")); + assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y")); + assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y")); + assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y")); + assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y")); + assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y")); + assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y")); + assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y")); + assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y")); + + assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y")); + assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y")); + assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y")); + + assertEquals(true, exec("def x = true; def y = true; return x & y")); + assertEquals(false, exec("def x = true; def y = false; return x & y")); + assertEquals(false, exec("def x = false; def y = true; return x & y")); + assertEquals(false, exec("def x = false; def y = false; return x & y")); + } + + public void testDefTypedLHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; def y = (byte)1; return x & y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; def y = (byte)1; return x & y"); + }); + assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("short x = (short)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("char x = (char)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("int x = (int)4; def y = (byte)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (byte)1; return x & y")); + + assertEquals(0, exec("byte x = (byte)4; def y = (short)1; return x & y")); + assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y")); + assertEquals(0, exec("char x = (char)4; def y = (short)1; return x & y")); + assertEquals(0, exec("int x = (int)4; def y = (short)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (short)1; return x & y")); + + assertEquals(0, exec("byte x = (byte)4; def y = (char)1; return x & y")); + assertEquals(0, exec("short x = (short)4; def y = (char)1; return x & y")); + assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y")); + assertEquals(0, exec("int x = (int)4; def y = (char)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (char)1; return x & y")); + + assertEquals(0, exec("byte x = (byte)4; def y = (int)1; return x & y")); + assertEquals(0, exec("short x = (short)4; def y = (int)1; return x & y")); + assertEquals(0, exec("char x = (char)4; def y = (int)1; return x & y")); + assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (int)1; return x & y")); + + assertEquals(0L, exec("byte x = (byte)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("short x = (short)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("char x = (char)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("int x = (int)4; def y = (long)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y")); + + assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y")); + assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y")); + assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y")); + assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y")); + assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y")); + + assertEquals(true, exec("boolean x = true; def y = true; return x & y")); + assertEquals(false, exec("boolean x = true; def y = false; return x & y")); + assertEquals(false, exec("boolean x = false; def y = true; return x & y")); + assertEquals(false, exec("boolean x = false; def y = false; return x & y")); + } + + public void testDefTypedRHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; byte y = (byte)1; return x & y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; byte y = (byte)1; return x & y"); + }); + assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (short)4; byte y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (char)4; byte y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (int)4; byte y = (byte)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; byte y = (byte)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; short y = (short)1; return x & y")); + assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y")); + assertEquals(0, exec("def x = (char)4; short y = (short)1; return x & y")); + assertEquals(0, exec("def x = (int)4; short y = (short)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; short y = (short)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; char y = (char)1; return x & y")); + assertEquals(0, exec("def x = (short)4; char y = (char)1; return x & y")); + assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y")); + assertEquals(0, exec("def x = (int)4; char y = (char)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; char y = (char)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; int y = (int)1; return x & y")); + assertEquals(0, exec("def x = (short)4; int y = (int)1; return x & y")); + assertEquals(0, exec("def x = (char)4; int y = (int)1; return x & y")); + assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; int y = (int)1; return x & y")); + + assertEquals(0L, exec("def x = (byte)4; long y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (short)4; long y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (char)4; long y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (int)4; long y = (long)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y")); + + assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y")); + assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y")); + assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y")); + assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y")); + assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y")); + + assertEquals(true, exec("def x = true; boolean y = true; return x & y")); + assertEquals(false, exec("def x = true; boolean y = false; return x & y")); + assertEquals(false, exec("def x = false; boolean y = true; return x & y")); + assertEquals(false, exec("def x = false; boolean y = false; return x & y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java index 8dabacdd5f9..e6a5537194d 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java @@ -70,9 +70,19 @@ public class ArrayTests extends ScriptTestCase { assertEquals(1, exec("int x; def y = new def[1]; x = y[0] = 1; return x;")); } + public void testArrayVariable() { + assertEquals(1, exec("int x = 1; int[] y = new int[x]; return y.length")); + } + public void testForLoop() { assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+ "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;")); } + /** + * Make sure we don't try and convert the {@code /} after the {@code ]} into a regex.... + */ + public void testDivideArray() { + assertEquals(1, exec("def[] x = new def[1]; x[0] = 2; return x[0] / 2")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java index 2a8f3674ac1..f59a9209f4c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java @@ -68,6 +68,18 @@ public class BasicExpressionTests extends ScriptTestCase { "((Map)y).put(2, 3);\n" + "return x.get(2);\n")); } + + public void testIllegalDefCast() { + Exception exception = expectScriptThrows(ClassCastException.class, () -> { + exec("def x = 1.0; int y = x; return y;"); + }); + assertTrue(exception.getMessage().contains("cannot be cast")); + + exception = expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (short)1; byte y = x; return y;"); + }); + assertTrue(exception.getMessage().contains("cannot be cast")); + } public void testCat() { assertEquals("aaabbb", exec("return \"aaa\" + \"bbb\";")); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java index c392e86806e..49b9ac21970 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java @@ -134,7 +134,7 @@ public class BasicStatementTests extends ScriptTestCase { " String cat = ''; int total = 0;" + " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total")); } - + public void testIterableForEachStatementDef() { assertEquals(6, exec("def l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" + " for (int x : l) total += x; return total")); @@ -153,7 +153,7 @@ public class BasicStatementTests extends ScriptTestCase { assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" + " for (int[] j : i) total += j[0]; return total")); } - + public void testArrayForEachStatementDef() { assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" + " for (int x : a) total += x; return total")); @@ -214,4 +214,20 @@ public class BasicStatementTests extends ScriptTestCase { assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s;"))[1]); assertEquals(10, ((Map)exec("Map s = new HashMap(); s.put(\"x\", 10); return s;")).get("x")); } + + public void testLambdas() { + Exception exception = expectThrows(Exception.class, () -> { + exec("Math.max(2, p -> {p.doSomething();})"); + }); + assertTrue(exception.getCause().getMessage().contains("Lambda functions are not supported.")); + } + + public void testLastInBlockDoesntNeedSemi() { + // One statement in the block in case that is a special case + assertEquals(10, exec("def i = 1; if (i == 1) {return 10}")); + assertEquals(10, exec("def i = 1; if (i == 1) {return 10} else {return 12}")); + // Two statements in the block, in case that is the general case + assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10}")); + assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10} else {return 12}")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java new file mode 100644 index 00000000000..355a1223273 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java @@ -0,0 +1,442 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +public class ComparisonTests extends ScriptTestCase { + + public void testDefEq() { + assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y")); + assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y")); + assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y")); + assertEquals(true, exec("def x = (int)4; def y = (int)4; return x == y")); + assertEquals(false, exec("def x = (long)5; def y = (int)3; return x == y")); + assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y")); + assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y")); + + assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y")); + assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y")); + assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y")); + assertEquals(true, exec("def x = (int)4; def y = (double)4; return x == y")); + assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y")); + assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y")); + assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y")); + + assertEquals(false, exec("def x = false; def y = true; return x == y")); + assertEquals(false, exec("def x = true; def y = false; return x == y")); + assertEquals(false, exec("def x = true; def y = null; return x == y")); + assertEquals(false, exec("def x = null; def y = true; return x == y")); + assertEquals(true, exec("def x = true; def y = true; return x == y")); + assertEquals(true, exec("def x = false; def y = false; return x == y")); + + assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y")); + assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y")); + } + + public void testDefEqTypedLHS() { + assertEquals(true, exec("byte x = (byte)7; def y = (int)7; return x == y")); + assertEquals(true, exec("short x = (short)6; def y = (int)6; return x == y")); + assertEquals(true, exec("char x = (char)5; def y = (int)5; return x == y")); + assertEquals(true, exec("int x = (int)4; def y = (int)4; return x == y")); + assertEquals(false, exec("long x = (long)5; def y = (int)3; return x == y")); + assertEquals(false, exec("float x = (float)6; def y = (int)2; return x == y")); + assertEquals(false, exec("double x = (double)7; def y = (int)1; return x == y")); + + assertEquals(true, exec("byte x = (byte)7; def y = (double)7; return x == y")); + assertEquals(true, exec("short x = (short)6; def y = (double)6; return x == y")); + assertEquals(true, exec("char x = (char)5; def y = (double)5; return x == y")); + assertEquals(true, exec("int x = (int)4; def y = (double)4; return x == y")); + assertEquals(false, exec("long x = (long)5; def y = (double)3; return x == y")); + assertEquals(false, exec("float x = (float)6; def y = (double)2; return x == y")); + assertEquals(false, exec("double x = (double)7; def y = (double)1; return x == y")); + + assertEquals(false, exec("boolean x = false; def y = true; return x == y")); + assertEquals(false, exec("boolean x = true; def y = false; return x == y")); + assertEquals(false, exec("boolean x = true; def y = null; return x == y")); + assertEquals(true, exec("boolean x = true; def y = true; return x == y")); + assertEquals(true, exec("boolean x = false; def y = false; return x == y")); + + assertEquals(true, exec("Map x = new HashMap(); def y = new HashMap(); return x == y")); + assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y")); + assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y")); + assertEquals(true, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y")); + } + + public void testDefEqTypedRHS() { + assertEquals(true, exec("def x = (byte)7; int y = (int)7; return x == y")); + assertEquals(true, exec("def x = (short)6; int y = (int)6; return x == y")); + assertEquals(true, exec("def x = (char)5; int y = (int)5; return x == y")); + assertEquals(true, exec("def x = (int)4; int y = (int)4; return x == y")); + assertEquals(false, exec("def x = (long)5; int y = (int)3; return x == y")); + assertEquals(false, exec("def x = (float)6; int y = (int)2; return x == y")); + assertEquals(false, exec("def x = (double)7; int y = (int)1; return x == y")); + + assertEquals(true, exec("def x = (byte)7; double y = (double)7; return x == y")); + assertEquals(true, exec("def x = (short)6; double y = (double)6; return x == y")); + assertEquals(true, exec("def x = (char)5; double y = (double)5; return x == y")); + assertEquals(true, exec("def x = (int)4; double y = (double)4; return x == y")); + assertEquals(false, exec("def x = (long)5; double y = (double)3; return x == y")); + assertEquals(false, exec("def x = (float)6; double y = (double)2; return x == y")); + assertEquals(false, exec("def x = (double)7; double y = (double)1; return x == y")); + + assertEquals(false, exec("def x = false; boolean y = true; return x == y")); + assertEquals(false, exec("def x = true; boolean y = false; return x == y")); + assertEquals(false, exec("def x = null; boolean y = true; return x == y")); + assertEquals(true, exec("def x = true; boolean y = true; return x == y")); + assertEquals(true, exec("def x = false; boolean y = false; return x == y")); + + assertEquals(true, exec("def x = new HashMap(); Map y = new HashMap(); return x == y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x == y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x == y")); + assertEquals(true, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x == y")); + } + + public void testDefEqr() { + assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y")); + assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y")); + assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y")); + assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y")); + assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y")); + assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y")); + assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y")); + assertEquals(false, exec("def x = false; def y = true; return x === y")); + + assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x === y")); + assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x === y")); + } + + public void testDefNe() { + assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y")); + assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y")); + assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y")); + assertEquals(false, exec("def x = (int)4; def y = (int)4; return x != y")); + assertEquals(true, exec("def x = (long)5; def y = (int)3; return x != y")); + assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y")); + assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y")); + + assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y")); + assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y")); + assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y")); + assertEquals(false, exec("def x = (int)4; def y = (double)4; return x != y")); + assertEquals(true, exec("def x = (long)5; def y = (double)3; return x != y")); + assertEquals(true, exec("def x = (float)6; def y = (double)2; return x != y")); + assertEquals(true, exec("def x = (double)7; def y = (double)1; return x != y")); + + assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x != y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y")); + assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y")); + + assertEquals(false, exec("def x = true; def y = true; return x != y")); + assertEquals(true, exec("def x = true; def y = false; return x != y")); + assertEquals(true, exec("def x = false; def y = true; return x != y")); + assertEquals(false, exec("def x = false; def y = false; return x != y")); + } + + public void testDefNeTypedLHS() { + assertEquals(false, exec("byte x = (byte)7; def y = (int)7; return x != y")); + assertEquals(false, exec("short x = (short)6; def y = (int)6; return x != y")); + assertEquals(false, exec("char x = (char)5; def y = (int)5; return x != y")); + assertEquals(false, exec("int x = (int)4; def y = (int)4; return x != y")); + assertEquals(true, exec("long x = (long)5; def y = (int)3; return x != y")); + assertEquals(true, exec("float x = (float)6; def y = (int)2; return x != y")); + assertEquals(true, exec("double x = (double)7; def y = (int)1; return x != y")); + + assertEquals(false, exec("byte x = (byte)7; def y = (double)7; return x != y")); + assertEquals(false, exec("short x = (short)6; def y = (double)6; return x != y")); + assertEquals(false, exec("char x = (char)5; def y = (double)5; return x != y")); + assertEquals(false, exec("int x = (int)4; def y = (double)4; return x != y")); + assertEquals(true, exec("long x = (long)5; def y = (double)3; return x != y")); + assertEquals(true, exec("float x = (float)6; def y = (double)2; return x != y")); + assertEquals(true, exec("double x = (double)7; def y = (double)1; return x != y")); + + assertEquals(false, exec("Map x = new HashMap(); def y = new HashMap(); return x != y")); + assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y")); + assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y")); + assertEquals(false, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y")); + + assertEquals(false, exec("boolean x = true; def y = true; return x != y")); + assertEquals(true, exec("boolean x = true; def y = false; return x != y")); + assertEquals(true, exec("boolean x = false; def y = true; return x != y")); + assertEquals(false, exec("boolean x = false; def y = false; return x != y")); + } + + public void testDefNeTypedRHS() { + assertEquals(false, exec("def x = (byte)7; int y = (int)7; return x != y")); + assertEquals(false, exec("def x = (short)6; int y = (int)6; return x != y")); + assertEquals(false, exec("def x = (char)5; int y = (int)5; return x != y")); + assertEquals(false, exec("def x = (int)4; int y = (int)4; return x != y")); + assertEquals(true, exec("def x = (long)5; int y = (int)3; return x != y")); + assertEquals(true, exec("def x = (float)6; int y = (int)2; return x != y")); + assertEquals(true, exec("def x = (double)7; int y = (int)1; return x != y")); + + assertEquals(false, exec("def x = (byte)7; double y = (double)7; return x != y")); + assertEquals(false, exec("def x = (short)6; double y = (double)6; return x != y")); + assertEquals(false, exec("def x = (char)5; double y = (double)5; return x != y")); + assertEquals(false, exec("def x = (int)4; double y = (double)4; return x != y")); + assertEquals(true, exec("def x = (long)5; double y = (double)3; return x != y")); + assertEquals(true, exec("def x = (float)6; double y = (double)2; return x != y")); + assertEquals(true, exec("def x = (double)7; double y = (double)1; return x != y")); + + assertEquals(false, exec("def x = new HashMap(); Map y = new HashMap(); return x != y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x != y")); + assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x != y")); + assertEquals(false, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x != y")); + + assertEquals(false, exec("def x = true; boolean y = true; return x != y")); + assertEquals(true, exec("def x = true; boolean y = false; return x != y")); + assertEquals(true, exec("def x = false; boolean y = true; return x != y")); + assertEquals(false, exec("def x = false; boolean y = false; return x != y")); + } + + public void testDefNer() { + assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y")); + assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y")); + assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y")); + assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y")); + assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y")); + assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y")); + assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y")); + + assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x !== y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x !== y")); + assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x !== y")); + assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x !== y")); + } + + public void testDefLt() { + assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x < y")); + assertEquals(true, exec("def x = (short)2; def y = (int)6; return x < y")); + assertEquals(true, exec("def x = (char)3; def y = (int)5; return x < y")); + assertEquals(false, exec("def x = (int)4; def y = (int)4; return x < y")); + assertEquals(false, exec("def x = (long)5; def y = (int)3; return x < y")); + assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y")); + assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y")); + + assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y")); + assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y")); + assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y")); + assertEquals(false, exec("def x = (int)4; def y = (double)4; return x < y")); + assertEquals(false, exec("def x = (long)5; def y = (double)3; return x < y")); + assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y")); + assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y")); + } + + public void testDefLtTypedLHS() { + assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x < y")); + assertEquals(true, exec("short x = (short)2; def y = (int)6; return x < y")); + assertEquals(true, exec("char x = (char)3; def y = (int)5; return x < y")); + assertEquals(false, exec("int x = (int)4; def y = (int)4; return x < y")); + assertEquals(false, exec("long x = (long)5; def y = (int)3; return x < y")); + assertEquals(false, exec("float x = (float)6; def y = (int)2; return x < y")); + assertEquals(false, exec("double x = (double)7; def y = (int)1; return x < y")); + + assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x < y")); + assertEquals(true, exec("short x = (short)2; def y = (double)6; return x < y")); + assertEquals(true, exec("char x = (char)3; def y = (double)5; return x < y")); + assertEquals(false, exec("int x = (int)4; def y = (double)4; return x < y")); + assertEquals(false, exec("long x = (long)5; def y = (double)3; return x < y")); + assertEquals(false, exec("float x = (float)6; def y = (double)2; return x < y")); + assertEquals(false, exec("double x = (double)7; def y = (double)1; return x < y")); + } + + public void testDefLtTypedRHS() { + assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x < y")); + assertEquals(true, exec("def x = (short)2; int y = (int)6; return x < y")); + assertEquals(true, exec("def x = (char)3; int y = (int)5; return x < y")); + assertEquals(false, exec("def x = (int)4; int y = (int)4; return x < y")); + assertEquals(false, exec("def x = (long)5; int y = (int)3; return x < y")); + assertEquals(false, exec("def x = (float)6; int y = (int)2; return x < y")); + assertEquals(false, exec("def x = (double)7; int y = (int)1; return x < y")); + + assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x < y")); + assertEquals(true, exec("def x = (short)2; double y = (double)6; return x < y")); + assertEquals(true, exec("def x = (char)3; double y = (double)5; return x < y")); + assertEquals(false, exec("def x = (int)4; double y = (double)4; return x < y")); + assertEquals(false, exec("def x = (long)5; double y = (double)3; return x < y")); + assertEquals(false, exec("def x = (float)6; double y = (double)2; return x < y")); + assertEquals(false, exec("def x = (double)7; double y = (double)1; return x < y")); + } + + public void testDefLte() { + assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y")); + assertEquals(true, exec("def x = (short)2; def y = (int)6; return x <= y")); + assertEquals(true, exec("def x = (char)3; def y = (int)5; return x <= y")); + assertEquals(true, exec("def x = (int)4; def y = (int)4; return x <= y")); + assertEquals(false, exec("def x = (long)5; def y = (int)3; return x <= y")); + assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y")); + assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y")); + + assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y")); + assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y")); + assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y")); + assertEquals(true, exec("def x = (int)4; def y = (double)4; return x <= y")); + assertEquals(false, exec("def x = (long)5; def y = (double)3; return x <= y")); + assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y")); + assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y")); + } + + public void testDefLteTypedLHS() { + assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x <= y")); + assertEquals(true, exec("short x = (short)2; def y = (int)6; return x <= y")); + assertEquals(true, exec("char x = (char)3; def y = (int)5; return x <= y")); + assertEquals(true, exec("int x = (int)4; def y = (int)4; return x <= y")); + assertEquals(false, exec("long x = (long)5; def y = (int)3; return x <= y")); + assertEquals(false, exec("float x = (float)6; def y = (int)2; return x <= y")); + assertEquals(false, exec("double x = (double)7; def y = (int)1; return x <= y")); + + assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x <= y")); + assertEquals(true, exec("short x = (short)2; def y = (double)6; return x <= y")); + assertEquals(true, exec("char x = (char)3; def y = (double)5; return x <= y")); + assertEquals(true, exec("int x = (int)4; def y = (double)4; return x <= y")); + assertEquals(false, exec("long x = (long)5; def y = (double)3; return x <= y")); + assertEquals(false, exec("float x = (float)6; def y = (double)2; return x <= y")); + assertEquals(false, exec("double x = (double)7; def y = (double)1; return x <= y")); + } + + public void testDefLteTypedRHS() { + assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x <= y")); + assertEquals(true, exec("def x = (short)2; int y = (int)6; return x <= y")); + assertEquals(true, exec("def x = (char)3; int y = (int)5; return x <= y")); + assertEquals(true, exec("def x = (int)4; int y = (int)4; return x <= y")); + assertEquals(false, exec("def x = (long)5; int y = (int)3; return x <= y")); + assertEquals(false, exec("def x = (float)6; int y = (int)2; return x <= y")); + assertEquals(false, exec("def x = (double)7; int y = (int)1; return x <= y")); + + assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x <= y")); + assertEquals(true, exec("def x = (short)2; double y = (double)6; return x <= y")); + assertEquals(true, exec("def x = (char)3; double y = (double)5; return x <= y")); + assertEquals(true, exec("def x = (int)4; double y = (double)4; return x <= y")); + assertEquals(false, exec("def x = (long)5; double y = (double)3; return x <= y")); + assertEquals(false, exec("def x = (float)6; double y = (double)2; return x <= y")); + assertEquals(false, exec("def x = (double)7; double y = (double)1; return x <= y")); + } + + public void testDefGt() { + assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y")); + assertEquals(false, exec("def x = (short)2; def y = (int)6; return x > y")); + assertEquals(false, exec("def x = (char)3; def y = (int)5; return x > y")); + assertEquals(false, exec("def x = (int)4; def y = (int)4; return x > y")); + assertEquals(true, exec("def x = (long)5; def y = (int)3; return x > y")); + assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y")); + assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y")); + + assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y")); + assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y")); + assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y")); + assertEquals(false, exec("def x = (int)4; def y = (double)4; return x > y")); + assertEquals(true, exec("def x = (long)5; def y = (double)3; return x > y")); + assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y")); + assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y")); + } + + public void testDefGtTypedLHS() { + assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x > y")); + assertEquals(false, exec("short x = (short)2; def y = (int)6; return x > y")); + assertEquals(false, exec("char x = (char)3; def y = (int)5; return x > y")); + assertEquals(false, exec("int x = (int)4; def y = (int)4; return x > y")); + assertEquals(true, exec("long x = (long)5; def y = (int)3; return x > y")); + assertEquals(true, exec("float x = (float)6; def y = (int)2; return x > y")); + assertEquals(true, exec("double x = (double)7; def y = (int)1; return x > y")); + + assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x > y")); + assertEquals(false, exec("short x = (short)2; def y = (double)6; return x > y")); + assertEquals(false, exec("char x = (char)3; def y = (double)5; return x > y")); + assertEquals(false, exec("int x = (int)4; def y = (double)4; return x > y")); + assertEquals(true, exec("long x = (long)5; def y = (double)3; return x > y")); + assertEquals(true, exec("float x = (float)6; def y = (double)2; return x > y")); + assertEquals(true, exec("double x = (double)7; def y = (double)1; return x > y")); + } + + public void testDefGtTypedRHS() { + assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x > y")); + assertEquals(false, exec("def x = (short)2; int y = (int)6; return x > y")); + assertEquals(false, exec("def x = (char)3; int y = (int)5; return x > y")); + assertEquals(false, exec("def x = (int)4; int y = (int)4; return x > y")); + assertEquals(true, exec("def x = (long)5; int y = (int)3; return x > y")); + assertEquals(true, exec("def x = (float)6; int y = (int)2; return x > y")); + assertEquals(true, exec("def x = (double)7; int y = (int)1; return x > y")); + + assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x > y")); + assertEquals(false, exec("def x = (short)2; double y = (double)6; return x > y")); + assertEquals(false, exec("def x = (char)3; double y = (double)5; return x > y")); + assertEquals(false, exec("def x = (int)4; double y = (double)4; return x > y")); + assertEquals(true, exec("def x = (long)5; double y = (double)3; return x > y")); + assertEquals(true, exec("def x = (float)6; double y = (double)2; return x > y")); + assertEquals(true, exec("def x = (double)7; double y = (double)1; return x > y")); + } + + public void testDefGte() { + assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y")); + assertEquals(false, exec("def x = (short)2; def y = (int)6; return x >= y")); + assertEquals(false, exec("def x = (char)3; def y = (int)5; return x >= y")); + assertEquals(true, exec("def x = (int)4; def y = (int)4; return x >= y")); + assertEquals(true, exec("def x = (long)5; def y = (int)3; return x >= y")); + assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y")); + assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y")); + + assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y")); + assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y")); + assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y")); + assertEquals(true, exec("def x = (int)4; def y = (double)4; return x >= y")); + assertEquals(true, exec("def x = (long)5; def y = (double)3; return x >= y")); + assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y")); + assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y")); + } + + public void testDefGteTypedLHS() { + assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x >= y")); + assertEquals(false, exec("short x = (short)2; def y = (int)6; return x >= y")); + assertEquals(false, exec("char x = (char)3; def y = (int)5; return x >= y")); + assertEquals(true, exec("int x = (int)4; def y = (int)4; return x >= y")); + assertEquals(true, exec("long x = (long)5; def y = (int)3; return x >= y")); + assertEquals(true, exec("float x = (float)6; def y = (int)2; return x >= y")); + assertEquals(true, exec("double x = (double)7; def y = (int)1; return x >= y")); + + assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x >= y")); + assertEquals(false, exec("short x = (short)2; def y = (double)6; return x >= y")); + assertEquals(false, exec("char x = (char)3; def y = (double)5; return x >= y")); + assertEquals(true, exec("int x = (int)4; def y = (double)4; return x >= y")); + assertEquals(true, exec("long x = (long)5; def y = (double)3; return x >= y")); + assertEquals(true, exec("float x = (float)6; def y = (double)2; return x >= y")); + assertEquals(true, exec("double x = (double)7; def y = (double)1; return x >= y")); + } + + public void testDefGteTypedRHS() { + assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x >= y")); + assertEquals(false, exec("def x = (short)2; int y = (int)6; return x >= y")); + assertEquals(false, exec("def x = (char)3; int y = (int)5; return x >= y")); + assertEquals(true, exec("def x = (int)4; int y = (int)4; return x >= y")); + assertEquals(true, exec("def x = (long)5; int y = (int)3; return x >= y")); + assertEquals(true, exec("def x = (float)6; int y = (int)2; return x >= y")); + assertEquals(true, exec("def x = (double)7; int y = (int)1; return x >= y")); + + assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x >= y")); + assertEquals(false, exec("def x = (short)2; double y = (double)6; return x >= y")); + assertEquals(false, exec("def x = (char)3; double y = (double)5; return x >= y")); + assertEquals(true, exec("def x = (int)4; double y = (double)4; return x >= y")); + assertEquals(true, exec("def x = (long)5; double y = (double)3; return x >= y")); + assertEquals(true, exec("def x = (float)6; double y = (double)2; return x >= y")); + assertEquals(true, exec("def x = (double)7; double y = (double)1; return x >= y")); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java index 7a55844a224..12104844e7c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java @@ -23,6 +23,8 @@ import java.lang.invoke.CallSite; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; +import java.util.Arrays; +import java.util.Collections; import org.elasticsearch.test.ESTestCase; @@ -33,7 +35,7 @@ public class DefBootstrapTests extends ESTestCase { CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), - DefBootstrap.METHOD_CALL, 0); + DefBootstrap.METHOD_CALL, 0L); MethodHandle handle = site.dynamicInvoker(); assertDepthEquals(site, 0); @@ -50,7 +52,7 @@ public class DefBootstrapTests extends ESTestCase { CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), - DefBootstrap.METHOD_CALL, 0); + DefBootstrap.METHOD_CALL, 0L); MethodHandle handle = site.dynamicInvoker(); assertDepthEquals(site, 0); @@ -72,7 +74,7 @@ public class DefBootstrapTests extends ESTestCase { CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), - DefBootstrap.METHOD_CALL, 0); + DefBootstrap.METHOD_CALL, 0L); MethodHandle handle = site.dynamicInvoker(); assertDepthEquals(site, 0); @@ -90,6 +92,19 @@ public class DefBootstrapTests extends ESTestCase { assertDepthEquals(site, 5); } + /** test that we really revert to a "generic" method that can handle any receiver types */ + public void testMegamorphic() throws Throwable { + DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), + "size", + MethodType.methodType(int.class, Object.class), + DefBootstrap.METHOD_CALL, 0L); + site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic + MethodHandle handle = site.dynamicInvoker(); + // arguments are cast to object here, or IDE compilers eat it :) + assertEquals(2, handle.invoke((Object) Arrays.asList("1", "2"))); + assertEquals(1, handle.invoke((Object) Collections.singletonMap("a", "b"))); + } + static void assertDepthEquals(CallSite site, int expected) { DefBootstrap.PIC dsite = (DefBootstrap.PIC) site; assertEquals(expected, dsite.depth); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java deleted file mode 100644 index ca611760f07..00000000000 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java +++ /dev/null @@ -1,926 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.painless; - -public class DefOperationTests extends ScriptTestCase { - public void testIllegalCast() { - Exception exception = expectScriptThrows(ClassCastException.class, () -> { - exec("def x = 1.0; int y = x; return y;"); - }); - assertTrue(exception.getMessage().contains("cannot be cast")); - - exception = expectScriptThrows(ClassCastException.class, () -> { - exec("def x = (short)1; byte y = x; return y;"); - }); - assertTrue(exception.getMessage().contains("cannot be cast")); - } - - public void testNot() { - assertEquals(~1, exec("def x = (byte)1; return ~x")); - assertEquals(~1, exec("def x = (short)1; return ~x")); - assertEquals(~1, exec("def x = (char)1; return ~x")); - assertEquals(~1, exec("def x = 1; return ~x")); - assertEquals(~1L, exec("def x = 1L; return ~x")); - } - - public void testNeg() { - assertEquals(-1, exec("def x = (byte)1; return -x")); - assertEquals(-1, exec("def x = (short)1; return -x")); - assertEquals(-1, exec("def x = (char)1; return -x")); - assertEquals(-1, exec("def x = 1; return -x")); - assertEquals(-1L, exec("def x = 1L; return -x")); - assertEquals(-1.0F, exec("def x = 1F; return -x")); - assertEquals(-1.0, exec("def x = 1.0; return -x")); - } - - public void testMul() { - assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y")); - assertEquals(4, exec("def x = (short)2; def y = (byte)2; return x * y")); - assertEquals(4, exec("def x = (char)2; def y = (byte)2; return x * y")); - assertEquals(4, exec("def x = (int)2; def y = (byte)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (byte)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (byte)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (byte)2; return x * y")); - - assertEquals(4, exec("def x = (byte)2; def y = (short)2; return x * y")); - assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y")); - assertEquals(4, exec("def x = (char)2; def y = (short)2; return x * y")); - assertEquals(4, exec("def x = (int)2; def y = (short)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (short)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (short)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (short)2; return x * y")); - - assertEquals(4, exec("def x = (byte)2; def y = (char)2; return x * y")); - assertEquals(4, exec("def x = (short)2; def y = (char)2; return x * y")); - assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y")); - assertEquals(4, exec("def x = (int)2; def y = (char)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (char)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (char)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (char)2; return x * y")); - - assertEquals(4, exec("def x = (byte)2; def y = (int)2; return x * y")); - assertEquals(4, exec("def x = (short)2; def y = (int)2; return x * y")); - assertEquals(4, exec("def x = (char)2; def y = (int)2; return x * y")); - assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (int)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (int)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (int)2; return x * y")); - - assertEquals(4L, exec("def x = (byte)2; def y = (long)2; return x * y")); - assertEquals(4L, exec("def x = (short)2; def y = (long)2; return x * y")); - assertEquals(4L, exec("def x = (char)2; def y = (long)2; return x * y")); - assertEquals(4L, exec("def x = (int)2; def y = (long)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (long)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (long)2; return x * y")); - - assertEquals(4F, exec("def x = (byte)2; def y = (float)2; return x * y")); - assertEquals(4F, exec("def x = (short)2; def y = (float)2; return x * y")); - assertEquals(4F, exec("def x = (char)2; def y = (float)2; return x * y")); - assertEquals(4F, exec("def x = (int)2; def y = (float)2; return x * y")); - assertEquals(4F, exec("def x = (long)2; def y = (float)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (float)2; return x * y")); - - assertEquals(4D, exec("def x = (byte)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (short)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (char)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (int)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (long)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y")); - - assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y")); - assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y")); - assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y")); - assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y")); - assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y")); - assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y")); - assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y")); - } - - public void testDiv() { - assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y")); - assertEquals(1, exec("def x = (short)2; def y = (byte)2; return x / y")); - assertEquals(1, exec("def x = (char)2; def y = (byte)2; return x / y")); - assertEquals(1, exec("def x = (int)2; def y = (byte)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (byte)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (byte)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (byte)2; return x / y")); - - assertEquals(1, exec("def x = (byte)2; def y = (short)2; return x / y")); - assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y")); - assertEquals(1, exec("def x = (char)2; def y = (short)2; return x / y")); - assertEquals(1, exec("def x = (int)2; def y = (short)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (short)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (short)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (short)2; return x / y")); - - assertEquals(1, exec("def x = (byte)2; def y = (char)2; return x / y")); - assertEquals(1, exec("def x = (short)2; def y = (char)2; return x / y")); - assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y")); - assertEquals(1, exec("def x = (int)2; def y = (char)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (char)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (char)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (char)2; return x / y")); - - assertEquals(1, exec("def x = (byte)2; def y = (int)2; return x / y")); - assertEquals(1, exec("def x = (short)2; def y = (int)2; return x / y")); - assertEquals(1, exec("def x = (char)2; def y = (int)2; return x / y")); - assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (int)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (int)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (int)2; return x / y")); - - assertEquals(1L, exec("def x = (byte)2; def y = (long)2; return x / y")); - assertEquals(1L, exec("def x = (short)2; def y = (long)2; return x / y")); - assertEquals(1L, exec("def x = (char)2; def y = (long)2; return x / y")); - assertEquals(1L, exec("def x = (int)2; def y = (long)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (long)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (long)2; return x / y")); - - assertEquals(1F, exec("def x = (byte)2; def y = (float)2; return x / y")); - assertEquals(1F, exec("def x = (short)2; def y = (float)2; return x / y")); - assertEquals(1F, exec("def x = (char)2; def y = (float)2; return x / y")); - assertEquals(1F, exec("def x = (int)2; def y = (float)2; return x / y")); - assertEquals(1F, exec("def x = (long)2; def y = (float)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (float)2; return x / y")); - - assertEquals(1D, exec("def x = (byte)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (short)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (char)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (int)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (long)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y")); - - assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y")); - assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y")); - assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y")); - assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y")); - assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y")); - assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y")); - assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y")); - } - - public void testRem() { - assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y")); - assertEquals(0, exec("def x = (short)2; def y = (byte)2; return x % y")); - assertEquals(0, exec("def x = (char)2; def y = (byte)2; return x % y")); - assertEquals(0, exec("def x = (int)2; def y = (byte)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (byte)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (byte)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (byte)2; return x % y")); - - assertEquals(0, exec("def x = (byte)2; def y = (short)2; return x % y")); - assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y")); - assertEquals(0, exec("def x = (char)2; def y = (short)2; return x % y")); - assertEquals(0, exec("def x = (int)2; def y = (short)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (short)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (short)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (short)2; return x % y")); - - assertEquals(0, exec("def x = (byte)2; def y = (char)2; return x % y")); - assertEquals(0, exec("def x = (short)2; def y = (char)2; return x % y")); - assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y")); - assertEquals(0, exec("def x = (int)2; def y = (char)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (char)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (char)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (char)2; return x % y")); - - assertEquals(0, exec("def x = (byte)2; def y = (int)2; return x % y")); - assertEquals(0, exec("def x = (short)2; def y = (int)2; return x % y")); - assertEquals(0, exec("def x = (char)2; def y = (int)2; return x % y")); - assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (int)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (int)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (int)2; return x % y")); - - assertEquals(0L, exec("def x = (byte)2; def y = (long)2; return x % y")); - assertEquals(0L, exec("def x = (short)2; def y = (long)2; return x % y")); - assertEquals(0L, exec("def x = (char)2; def y = (long)2; return x % y")); - assertEquals(0L, exec("def x = (int)2; def y = (long)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (long)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (long)2; return x % y")); - - assertEquals(0F, exec("def x = (byte)2; def y = (float)2; return x % y")); - assertEquals(0F, exec("def x = (short)2; def y = (float)2; return x % y")); - assertEquals(0F, exec("def x = (char)2; def y = (float)2; return x % y")); - assertEquals(0F, exec("def x = (int)2; def y = (float)2; return x % y")); - assertEquals(0F, exec("def x = (long)2; def y = (float)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (float)2; return x % y")); - - assertEquals(0D, exec("def x = (byte)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (short)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (char)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (int)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (long)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y")); - - assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y")); - assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y")); - assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y")); - assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y")); - assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y")); - assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y")); - assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y")); - } - - public void testAdd() { - assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y")); - assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y")); - assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x + y")); - assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (byte)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (byte)1; return x + y")); - - assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x + y")); - assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y")); - assertEquals(2, exec("def x = (char)1; def y = (short)1; return x + y")); - assertEquals(2, exec("def x = (int)1; def y = (short)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (short)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (short)1; return x + y")); - - assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x + y")); - assertEquals(2, exec("def x = (short)1; def y = (char)1; return x + y")); - assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y")); - assertEquals(2, exec("def x = (int)1; def y = (char)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (char)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (char)1; return x + y")); - - assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x + y")); - assertEquals(2, exec("def x = (short)1; def y = (int)1; return x + y")); - assertEquals(2, exec("def x = (char)1; def y = (int)1; return x + y")); - assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (int)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (int)1; return x + y")); - - assertEquals(2L, exec("def x = (byte)1; def y = (long)1; return x + y")); - assertEquals(2L, exec("def x = (short)1; def y = (long)1; return x + y")); - assertEquals(2L, exec("def x = (char)1; def y = (long)1; return x + y")); - assertEquals(2L, exec("def x = (int)1; def y = (long)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (long)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (long)1; return x + y")); - - assertEquals(2F, exec("def x = (byte)1; def y = (float)1; return x + y")); - assertEquals(2F, exec("def x = (short)1; def y = (float)1; return x + y")); - assertEquals(2F, exec("def x = (char)1; def y = (float)1; return x + y")); - assertEquals(2F, exec("def x = (int)1; def y = (float)1; return x + y")); - assertEquals(2F, exec("def x = (long)1; def y = (float)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (float)1; return x + y")); - - assertEquals(2D, exec("def x = (byte)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (short)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (char)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (int)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y")); - - assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y")); - assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y")); - assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y")); - assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y")); - assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y")); - assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y")); - assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y")); - } - - public void testSub() { - assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y")); - assertEquals(0, exec("def x = (short)1; def y = (byte)1; return x - y")); - assertEquals(0, exec("def x = (char)1; def y = (byte)1; return x - y")); - assertEquals(0, exec("def x = (int)1; def y = (byte)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (byte)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (byte)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (byte)1; return x - y")); - - assertEquals(0, exec("def x = (byte)1; def y = (short)1; return x - y")); - assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y")); - assertEquals(0, exec("def x = (char)1; def y = (short)1; return x - y")); - assertEquals(0, exec("def x = (int)1; def y = (short)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (short)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (short)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (short)1; return x - y")); - - assertEquals(0, exec("def x = (byte)1; def y = (char)1; return x - y")); - assertEquals(0, exec("def x = (short)1; def y = (char)1; return x - y")); - assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y")); - assertEquals(0, exec("def x = (int)1; def y = (char)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (char)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (char)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (char)1; return x - y")); - - assertEquals(0, exec("def x = (byte)1; def y = (int)1; return x - y")); - assertEquals(0, exec("def x = (short)1; def y = (int)1; return x - y")); - assertEquals(0, exec("def x = (char)1; def y = (int)1; return x - y")); - assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (int)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (int)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (int)1; return x - y")); - - assertEquals(0L, exec("def x = (byte)1; def y = (long)1; return x - y")); - assertEquals(0L, exec("def x = (short)1; def y = (long)1; return x - y")); - assertEquals(0L, exec("def x = (char)1; def y = (long)1; return x - y")); - assertEquals(0L, exec("def x = (int)1; def y = (long)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (long)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (long)1; return x - y")); - - assertEquals(0F, exec("def x = (byte)1; def y = (float)1; return x - y")); - assertEquals(0F, exec("def x = (short)1; def y = (float)1; return x - y")); - assertEquals(0F, exec("def x = (char)1; def y = (float)1; return x - y")); - assertEquals(0F, exec("def x = (int)1; def y = (float)1; return x - y")); - assertEquals(0F, exec("def x = (long)1; def y = (float)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (float)1; return x - y")); - - assertEquals(0D, exec("def x = (byte)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (short)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (char)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (int)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y")); - - assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y")); - assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y")); - assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y")); - assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y")); - assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y")); - assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y")); - assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y")); - } - - public void testLsh() { - assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (byte)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (byte)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (short)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (short)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (char)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (char)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (int)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (int)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (long)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (long)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (float)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (float)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (float)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (float)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (float)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (float)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (double)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (double)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (double)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (double)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (double)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (double)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y")); - - assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y")); - assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y")); - assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y")); - assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y")); - assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y")); - assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y")); - assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y")); - } - - public void testRsh() { - assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y")); - assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y")); - assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y")); - assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y")); - assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y")); - assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y")); - assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y")); - } - - public void testUsh() { - assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y")); - - assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y")); - assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y")); - assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y")); - assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y")); - assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y")); - assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y")); - assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y")); - } - - public void testAnd() { - assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y")); - assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y")); - assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y")); - assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (byte)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (byte)1; return x & y")); - - assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y")); - assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y")); - assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y")); - assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (short)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (short)1; return x & y")); - - assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y")); - assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y")); - assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y")); - assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (char)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (char)1; return x & y")); - - assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y")); - assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y")); - assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y")); - assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (int)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (int)1; return x & y")); - - assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (long)1; return x & y")); - - assertEquals(0L, exec("def x = (byte)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (short)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (char)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (int)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (float)1; return x & y")); - - assertEquals(0L, exec("def x = (byte)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (short)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (char)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (int)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (double)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y")); - - assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y")); - assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y")); - assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y")); - assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y")); - assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y")); - assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y")); - assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y")); - } - - public void testXor() { - assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); - assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y")); - assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y")); - assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x ^ y")); - - assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y")); - assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); - assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y")); - assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x ^ y")); - - assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y")); - assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y")); - assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); - assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x ^ y")); - - assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y")); - assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y")); - assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y")); - assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x ^ y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x ^ y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x ^ y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y")); - - assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); - assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); - assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); - assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); - assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); - assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y")); - assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y")); - } - - public void testOr() { - assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y")); - assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y")); - assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y")); - assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x | y")); - - assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y")); - assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y")); - assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y")); - assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x | y")); - - assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y")); - assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y")); - assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y")); - assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x | y")); - - assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y")); - assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y")); - assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y")); - assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x | y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x | y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x | y")); - - assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y")); - - assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y")); - assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y")); - assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y")); - assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y")); - assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y")); - assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y")); - assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y")); - } - - public void testEq() { - assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y")); - assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y")); - assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y")); - assertEquals(true, exec("def x = (int)4; def y = (int)4; return x == y")); - assertEquals(false, exec("def x = (long)5; def y = (int)3; return x == y")); - assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y")); - assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y")); - - assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y")); - assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y")); - assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y")); - assertEquals(true, exec("def x = (int)4; def y = (double)4; return x == y")); - assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y")); - assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y")); - assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y")); - - assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y")); - assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y")); - assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y")); - assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y")); - } - - public void testEqr() { - assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y")); - assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y")); - assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y")); - assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y")); - assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y")); - assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y")); - assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y")); - - assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y")); - assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y")); - assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x === y")); - assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x === y")); - } - - public void testNe() { - assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y")); - assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y")); - assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y")); - assertEquals(false, exec("def x = (int)4; def y = (int)4; return x != y")); - assertEquals(true, exec("def x = (long)5; def y = (int)3; return x != y")); - assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y")); - assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y")); - - assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y")); - assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y")); - assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y")); - assertEquals(false, exec("def x = (int)4; def y = (double)4; return x != y")); - assertEquals(true, exec("def x = (long)5; def y = (double)3; return x != y")); - assertEquals(true, exec("def x = (float)6; def y = (double)2; return x != y")); - assertEquals(true, exec("def x = (double)7; def y = (double)1; return x != y")); - - assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x != y")); - assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y")); - assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y")); - assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y")); - } - - public void testNer() { - assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y")); - assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y")); - assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y")); - assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y")); - assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y")); - assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y")); - assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y")); - - assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x !== y")); - assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x !== y")); - assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x !== y")); - assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x !== y")); - } - - public void testLt() { - assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x < y")); - assertEquals(true, exec("def x = (short)2; def y = (int)6; return x < y")); - assertEquals(true, exec("def x = (char)3; def y = (int)5; return x < y")); - assertEquals(false, exec("def x = (int)4; def y = (int)4; return x < y")); - assertEquals(false, exec("def x = (long)5; def y = (int)3; return x < y")); - assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y")); - assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y")); - - assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y")); - assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y")); - assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y")); - assertEquals(false, exec("def x = (int)4; def y = (double)4; return x < y")); - assertEquals(false, exec("def x = (long)5; def y = (double)3; return x < y")); - assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y")); - assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y")); - } - - public void testLte() { - assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y")); - assertEquals(true, exec("def x = (short)2; def y = (int)6; return x <= y")); - assertEquals(true, exec("def x = (char)3; def y = (int)5; return x <= y")); - assertEquals(true, exec("def x = (int)4; def y = (int)4; return x <= y")); - assertEquals(false, exec("def x = (long)5; def y = (int)3; return x <= y")); - assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y")); - assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y")); - - assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y")); - assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y")); - assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y")); - assertEquals(true, exec("def x = (int)4; def y = (double)4; return x <= y")); - assertEquals(false, exec("def x = (long)5; def y = (double)3; return x <= y")); - assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y")); - assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y")); - } - - public void testGt() { - assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y")); - assertEquals(false, exec("def x = (short)2; def y = (int)6; return x > y")); - assertEquals(false, exec("def x = (char)3; def y = (int)5; return x > y")); - assertEquals(false, exec("def x = (int)4; def y = (int)4; return x > y")); - assertEquals(true, exec("def x = (long)5; def y = (int)3; return x > y")); - assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y")); - assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y")); - - assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y")); - assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y")); - assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y")); - assertEquals(false, exec("def x = (int)4; def y = (double)4; return x > y")); - assertEquals(true, exec("def x = (long)5; def y = (double)3; return x > y")); - assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y")); - assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y")); - } - - public void testGte() { - assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y")); - assertEquals(false, exec("def x = (short)2; def y = (int)6; return x >= y")); - assertEquals(false, exec("def x = (char)3; def y = (int)5; return x >= y")); - assertEquals(true, exec("def x = (int)4; def y = (int)4; return x >= y")); - assertEquals(true, exec("def x = (long)5; def y = (int)3; return x >= y")); - assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y")); - assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y")); - - assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y")); - assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y")); - assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y")); - assertEquals(true, exec("def x = (int)4; def y = (double)4; return x >= y")); - assertEquals(true, exec("def x = (long)5; def y = (double)3; return x >= y")); - assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y")); - assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y")); - } -} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java index f9771862335..1dd0d09a580 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java @@ -178,4 +178,244 @@ public class DefOptimizationTests extends ScriptTestCase { }); assertTrue(exception.getMessage().contains("Cannot cast java.lang.Double to java.lang.Integer")); } + + public void testMulOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x * y", + "INVOKEDYNAMIC mul(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testMulOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x * y", + "INVOKEDYNAMIC mul(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testMulOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x * y", + "INVOKEDYNAMIC mul(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testDivOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x / y", + "INVOKEDYNAMIC div(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testDivOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x / y", + "INVOKEDYNAMIC div(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testDivOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x / y", + "INVOKEDYNAMIC div(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testRemOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x % y", + "INVOKEDYNAMIC rem(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testRemOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x % y", + "INVOKEDYNAMIC rem(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testRemOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x % y", + "INVOKEDYNAMIC rem(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testAddOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x + y", + "INVOKEDYNAMIC add(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testAddOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x + y", + "INVOKEDYNAMIC add(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testAddOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x + y", + "INVOKEDYNAMIC add(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testSubOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x - y", + "INVOKEDYNAMIC sub(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testSubOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x - y", + "INVOKEDYNAMIC sub(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testSubOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x - y", + "INVOKEDYNAMIC sub(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testLshOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x << y", + "INVOKEDYNAMIC lsh(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testLshOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x << y", + "INVOKEDYNAMIC lsh(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testLshOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x << y", + "INVOKEDYNAMIC lsh(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testRshOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x >> y", + "INVOKEDYNAMIC rsh(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testRshOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x >> y", + "INVOKEDYNAMIC rsh(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testRshOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x >> y", + "INVOKEDYNAMIC rsh(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testUshOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x >>> y", + "INVOKEDYNAMIC ush(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testUshOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x >>> y", + "INVOKEDYNAMIC ush(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testUshOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x >>> y", + "INVOKEDYNAMIC ush(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testAndOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x & y", + "INVOKEDYNAMIC and(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testAndOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x & y", + "INVOKEDYNAMIC and(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testAndOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x & y", + "INVOKEDYNAMIC and(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testOrOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x | y", + "INVOKEDYNAMIC or(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testOrOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x | y", + "INVOKEDYNAMIC or(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testOrOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x | y", + "INVOKEDYNAMIC or(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testXorOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x ^ y", + "INVOKEDYNAMIC xor(ILjava/lang/Object;)Ljava/lang/Object;"); + } + + public void testXorOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x ^ y", + "INVOKEDYNAMIC xor(Ljava/lang/Object;I)Ljava/lang/Object;"); + } + + public void testXorOptRet() { + assertBytecodeExists("def x = 1; def y = 2; double d = x ^ y", + "INVOKEDYNAMIC xor(Ljava/lang/Object;Ljava/lang/Object;)D"); + } + + public void testLtOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x < y", + "INVOKEDYNAMIC lt(ILjava/lang/Object;)Z"); + } + + public void testLtOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x < y", + "INVOKEDYNAMIC lt(Ljava/lang/Object;I)Z"); + } + + public void testLteOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x <= y", + "INVOKEDYNAMIC lte(ILjava/lang/Object;)Z"); + } + + public void testLteOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x <= y", + "INVOKEDYNAMIC lte(Ljava/lang/Object;I)Z"); + } + + public void testEqOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x == y", + "INVOKEDYNAMIC eq(ILjava/lang/Object;)Z"); + } + + public void testEqOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x == y", + "INVOKEDYNAMIC eq(Ljava/lang/Object;I)Z"); + } + + public void testNeqOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x != y", + "INVOKEDYNAMIC eq(ILjava/lang/Object;)Z"); + } + + public void testNeqOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x != y", + "INVOKEDYNAMIC eq(Ljava/lang/Object;I)Z"); + } + + public void testGteOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x >= y", + "INVOKEDYNAMIC gte(ILjava/lang/Object;)Z"); + } + + public void testGteOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x >= y", + "INVOKEDYNAMIC gte(Ljava/lang/Object;I)Z"); + } + + public void testGtOptLHS() { + assertBytecodeExists("int x = 1; def y = 2; return x > y", + "INVOKEDYNAMIC gt(ILjava/lang/Object;)Z"); + } + + public void testGtOptRHS() { + assertBytecodeExists("def x = 1; int y = 2; return x > y", + "INVOKEDYNAMIC gt(Ljava/lang/Object;I)Z"); + } + + public void testUnaryMinusOptRet() { + assertBytecodeExists("def x = 1; double y = -x; return y", + "INVOKEDYNAMIC neg(Ljava/lang/Object;)D"); + } + + public void testUnaryNotOptRet() { + assertBytecodeExists("def x = 1; double y = ~x; return y", + "INVOKEDYNAMIC not(Ljava/lang/Object;)D"); + } + + public void testUnaryPlusOptRet() { + assertBytecodeExists("def x = 1; double y = +x; return y", + "INVOKEDYNAMIC plus(Ljava/lang/Object;)D"); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java index 834f998723e..226cf6f1710 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java @@ -24,6 +24,11 @@ package org.elasticsearch.painless; public class DivisionTests extends ScriptTestCase { // TODO: byte,short,char + + public void testBasics() throws Exception { + assertEquals(2.25F / 1.5F, exec("return 2.25F / 1.5F;")); + assertEquals(0.5, exec("double x = 1; float y = 2; return x / y;")); + } public void testInt() throws Exception { assertEquals(1/1, exec("int x = 1; int y = 1; return x/y;")); @@ -132,4 +137,202 @@ public class DivisionTests extends ScriptTestCase { exec("return 1L/0L;"); }); } + + public void testDef() { + assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (short)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (char)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (int)2; def y = (byte)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (byte)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (byte)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (byte)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; def y = (short)2; return x / y")); + assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y")); + assertEquals(1, exec("def x = (char)2; def y = (short)2; return x / y")); + assertEquals(1, exec("def x = (int)2; def y = (short)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (short)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (short)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (short)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; def y = (char)2; return x / y")); + assertEquals(1, exec("def x = (short)2; def y = (char)2; return x / y")); + assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y")); + assertEquals(1, exec("def x = (int)2; def y = (char)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (char)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (char)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (char)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; def y = (int)2; return x / y")); + assertEquals(1, exec("def x = (short)2; def y = (int)2; return x / y")); + assertEquals(1, exec("def x = (char)2; def y = (int)2; return x / y")); + assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (int)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (int)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (int)2; return x / y")); + + assertEquals(1L, exec("def x = (byte)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (short)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (char)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (int)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (long)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (long)2; return x / y")); + + assertEquals(1F, exec("def x = (byte)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (short)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (char)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (int)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (long)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (float)2; return x / y")); + + assertEquals(1D, exec("def x = (byte)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (short)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (char)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (int)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (long)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y")); + assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y")); + assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y")); + } + + public void testDefTypedLHS() { + assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("short x = (short)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("char x = (char)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("int x = (int)2; def y = (byte)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (byte)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (byte)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (byte)2; return x / y")); + + assertEquals(1, exec("byte x = (byte)2; def y = (short)2; return x / y")); + assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y")); + assertEquals(1, exec("char x = (char)2; def y = (short)2; return x / y")); + assertEquals(1, exec("int x = (int)2; def y = (short)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (short)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (short)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (short)2; return x / y")); + + assertEquals(1, exec("byte x = (byte)2; def y = (char)2; return x / y")); + assertEquals(1, exec("short x = (short)2; def y = (char)2; return x / y")); + assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y")); + assertEquals(1, exec("int x = (int)2; def y = (char)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (char)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (char)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (char)2; return x / y")); + + assertEquals(1, exec("byte x = (byte)2; def y = (int)2; return x / y")); + assertEquals(1, exec("short x = (short)2; def y = (int)2; return x / y")); + assertEquals(1, exec("char x = (char)2; def y = (int)2; return x / y")); + assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (int)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (int)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (int)2; return x / y")); + + assertEquals(1L, exec("byte x = (byte)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("short x = (short)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("char x = (char)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("int x = (int)2; def y = (long)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (long)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (long)2; return x / y")); + + assertEquals(1F, exec("byte x = (byte)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("short x = (short)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("char x = (char)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("int x = (int)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("long x = (long)2; def y = (float)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (float)2; return x / y")); + + assertEquals(1D, exec("byte x = (byte)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("short x = (short)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("char x = (char)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("int x = (int)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("long x = (long)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("float x = (float)2; def y = (double)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y")); + + assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y")); + assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y")); + assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y")); + assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y")); + assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y")); + assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y")); + assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y")); + } + + public void testDefTypedRHS() { + assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (short)2; byte y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (char)2; byte y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (int)2; byte y = (byte)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; byte y = (byte)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; byte y = (byte)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; byte y = (byte)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; short y = (short)2; return x / y")); + assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y")); + assertEquals(1, exec("def x = (char)2; short y = (short)2; return x / y")); + assertEquals(1, exec("def x = (int)2; short y = (short)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; short y = (short)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; short y = (short)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; short y = (short)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; char y = (char)2; return x / y")); + assertEquals(1, exec("def x = (short)2; char y = (char)2; return x / y")); + assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y")); + assertEquals(1, exec("def x = (int)2; char y = (char)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; char y = (char)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; char y = (char)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; char y = (char)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; int y = (int)2; return x / y")); + assertEquals(1, exec("def x = (short)2; int y = (int)2; return x / y")); + assertEquals(1, exec("def x = (char)2; int y = (int)2; return x / y")); + assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; int y = (int)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; int y = (int)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; int y = (int)2; return x / y")); + + assertEquals(1L, exec("def x = (byte)2; long y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (short)2; long y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (char)2; long y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (int)2; long y = (long)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; long y = (long)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; long y = (long)2; return x / y")); + + assertEquals(1F, exec("def x = (byte)2; float y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (short)2; float y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (char)2; float y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (int)2; float y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (long)2; float y = (float)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; float y = (float)2; return x / y")); + + assertEquals(1D, exec("def x = (byte)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (short)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (char)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (int)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (long)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (float)2; double y = (double)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y")); + + assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y")); + assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y")); + assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y")); + assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y")); + assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y")); + assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y")); + assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java index 12265f161ab..8ed2f3261cf 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java @@ -39,12 +39,12 @@ public class FunctionRefTests extends ScriptTestCase { public void testCtorMethodReference() { assertEquals(3.0D, - exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + - "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + - "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " + - "DoubleSummaryStatistics::accept, " + - "DoubleSummaryStatistics::combine); " + - "return stats.getSum()")); + exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + + "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + + "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " + + "DoubleSummaryStatistics::accept, " + + "DoubleSummaryStatistics::combine); " + + "return stats.getSum()")); } public void testCtorMethodReferenceDef() { @@ -57,6 +57,92 @@ public class FunctionRefTests extends ScriptTestCase { "return stats.getSum()")); } + public void testArrayCtorMethodRef() { + assertEquals(1.0D, + exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + + "def[] array = l.stream().toArray(Double[]::new);" + + "return array[0];")); + } + + public void testArrayCtorMethodRefDef() { + assertEquals(1.0D, + exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " + + "def[] array = l.stream().toArray(Double[]::new);" + + "return array[0];")); + } + + public void testCapturingMethodReference() { + assertEquals("5", exec("Integer x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);")); + assertEquals("[]", exec("List l = new ArrayList(); return Optional.empty().orElseGet(l::toString);")); + } + + public void testCapturingMethodReferenceDefImpl() { + assertEquals("5", exec("def x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);")); + assertEquals("[]", exec("def l = new ArrayList(); return Optional.empty().orElseGet(l::toString);")); + } + + public void testCapturingMethodReferenceDefInterface() { + assertEquals("5", exec("Integer x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);")); + assertEquals("[]", exec("List l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);")); + } + + public void testCapturingMethodReferenceDefEverywhere() { + assertEquals("5", exec("def x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);")); + assertEquals("[]", exec("def l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);")); + } + + public void testCapturingMethodReferenceMultipleLambdas() { + assertEquals("testingcdefg", exec( + "String x = 'testing';" + + "String y = 'abcdefg';" + + "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + + "return test.twoFunctionsOfX(x::concat, y::substring);")); + } + + public void testCapturingMethodReferenceMultipleLambdasDefImpls() { + assertEquals("testingcdefg", exec( + "def x = 'testing';" + + "def y = 'abcdefg';" + + "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + + "return test.twoFunctionsOfX(x::concat, y::substring);")); + } + + public void testCapturingMethodReferenceMultipleLambdasDefInterface() { + assertEquals("testingcdefg", exec( + "String x = 'testing';" + + "String y = 'abcdefg';" + + "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + + "return test.twoFunctionsOfX(x::concat, y::substring);")); + } + + public void testCapturingMethodReferenceMultipleLambdasDefEverywhere() { + assertEquals("testingcdefg", exec( + "def x = 'testing';" + + "def y = 'abcdefg';" + + "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + + "return test.twoFunctionsOfX(x::concat, y::substring);")); + } + + public void testOwnStaticMethodReference() { + assertEquals(2, exec("int mycompare(int i, int j) { j - i } " + + "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);")); + } + + public void testOwnStaticMethodReferenceDef() { + assertEquals(2, exec("int mycompare(int i, int j) { j - i } " + + "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);")); + } + + public void testInterfaceDefaultMethod() { + assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + + "Map map = new HashMap(); f(map::getOrDefault)")); + } + + public void testInterfaceDefaultMethodDef() { + assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + + "def map = new HashMap(); f(map::getOrDefault)")); + } + public void testMethodMissing() { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::bogus); return l.get(0);"); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java new file mode 100644 index 00000000000..48c09cd4025 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +public class FunctionTests extends ScriptTestCase { + public void testBasic() { + assertEquals(5, exec("int get() {5;} get()")); + } + + public void testReference() { + assertEquals(5, exec("void get(int[] x) {x[0] = 5;} int[] y = new int[1]; y[0] = 1; get(y); y[0]")); + } + + public void testConcat() { + assertEquals("xyxy", exec("String catcat(String single) {single + single;} catcat('xy')")); + } + + public void testMultiArgs() { + assertEquals(5, exec("int add(int x, int y) {return x + y;} int x = 1, y = 2; add(add(x, x), add(x, y))")); + } + + public void testMultiFuncs() { + assertEquals(1, exec("int add(int x, int y) {return x + y;} int sub(int x, int y) {return x - y;} add(2, sub(3, 4))")); + assertEquals(3, exec("int sub2(int x, int y) {sub(x, y) - y;} int sub(int x, int y) {return x - y;} sub2(5, 1)")); + } + + public void testRecursion() { + assertEquals(55, exec("int fib(int n) {if (n <= 1) return n; else return fib(n-1) + fib(n-2);} fib(10)")); + } + + public void testEmpty() { + Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> { + exec("void test(int x) {} test()"); + }); + assertTrue(expected.getMessage().contains("Cannot generate an empty function")); + } + + public void testDuplicates() { + Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> { + exec("void test(int x) {x = 2;} void test(def y) {y = 3;} test()"); + }); + assertTrue(expected.getMessage().contains("Duplicate functions")); + } + + public void testInfiniteLoop() { + Error expected = expectScriptThrows(PainlessError.class, () -> { + exec("void test() {boolean x = true; while (x) {}} test()"); + }); + assertTrue(expected.getMessage().contains( + "The maximum number of statements that can be executed in a loop has been reached.")); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java index 9c10d90bec2..2ccb51d7946 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java @@ -24,6 +24,10 @@ package org.elasticsearch.painless; public class MultiplicationTests extends ScriptTestCase { // TODO: short,byte,char + + public void testBasics() throws Exception { + assertEquals(8, exec("int x = 4; char y = 2; return x*y;")); + } public void testInt() throws Exception { assertEquals(1*1, exec("int x = 1; int y = 1; return x*y;")); @@ -123,4 +127,202 @@ public class MultiplicationTests extends ScriptTestCase { assertEquals(10.0*0.0, exec("return 10.0*0.0;")); assertEquals(0.0*0.0, exec("return 0.0*0.0;")); } + + public void testDef() { + assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (short)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (char)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (int)2; def y = (byte)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (byte)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (byte)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (byte)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; def y = (short)2; return x * y")); + assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y")); + assertEquals(4, exec("def x = (char)2; def y = (short)2; return x * y")); + assertEquals(4, exec("def x = (int)2; def y = (short)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (short)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (short)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (short)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; def y = (char)2; return x * y")); + assertEquals(4, exec("def x = (short)2; def y = (char)2; return x * y")); + assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y")); + assertEquals(4, exec("def x = (int)2; def y = (char)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (char)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (char)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (char)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; def y = (int)2; return x * y")); + assertEquals(4, exec("def x = (short)2; def y = (int)2; return x * y")); + assertEquals(4, exec("def x = (char)2; def y = (int)2; return x * y")); + assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (int)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (int)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (int)2; return x * y")); + + assertEquals(4L, exec("def x = (byte)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (short)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (char)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (int)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (long)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (long)2; return x * y")); + + assertEquals(4F, exec("def x = (byte)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (short)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (char)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (int)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (long)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (float)2; return x * y")); + + assertEquals(4D, exec("def x = (byte)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (short)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (char)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (int)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (long)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y")); + assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y")); + assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y")); + } + + public void testDefTypedLHS() { + assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("short x = (short)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("char x = (char)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("int x = (int)2; def y = (byte)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (byte)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (byte)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (byte)2; return x * y")); + + assertEquals(4, exec("byte x = (byte)2; def y = (short)2; return x * y")); + assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y")); + assertEquals(4, exec("char x = (char)2; def y = (short)2; return x * y")); + assertEquals(4, exec("int x = (int)2; def y = (short)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (short)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (short)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (short)2; return x * y")); + + assertEquals(4, exec("byte x = (byte)2; def y = (char)2; return x * y")); + assertEquals(4, exec("short x = (short)2; def y = (char)2; return x * y")); + assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y")); + assertEquals(4, exec("int x = (int)2; def y = (char)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (char)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (char)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (char)2; return x * y")); + + assertEquals(4, exec("byte x = (byte)2; def y = (int)2; return x * y")); + assertEquals(4, exec("short x = (short)2; def y = (int)2; return x * y")); + assertEquals(4, exec("char x = (char)2; def y = (int)2; return x * y")); + assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (int)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (int)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (int)2; return x * y")); + + assertEquals(4L, exec("byte x = (byte)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("short x = (short)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("char x = (char)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("int x = (int)2; def y = (long)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (long)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (long)2; return x * y")); + + assertEquals(4F, exec("byte x = (byte)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("short x = (short)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("char x = (char)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("int x = (int)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("long x = (long)2; def y = (float)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (float)2; return x * y")); + + assertEquals(4D, exec("byte x = (byte)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("short x = (short)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("char x = (char)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("int x = (int)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("long x = (long)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("float x = (float)2; def y = (double)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y")); + + assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y")); + assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y")); + assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y")); + assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y")); + assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y")); + assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y")); + assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y")); + } + + public void testDefTypedRHS() { + assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (short)2; byte y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (char)2; byte y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (int)2; byte y = (byte)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; byte y = (byte)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; byte y = (byte)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; byte y = (byte)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; short y = (short)2; return x * y")); + assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y")); + assertEquals(4, exec("def x = (char)2; short y = (short)2; return x * y")); + assertEquals(4, exec("def x = (int)2; short y = (short)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; short y = (short)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; short y = (short)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; short y = (short)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; char y = (char)2; return x * y")); + assertEquals(4, exec("def x = (short)2; char y = (char)2; return x * y")); + assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y")); + assertEquals(4, exec("def x = (int)2; char y = (char)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; char y = (char)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; char y = (char)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; char y = (char)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; int y = (int)2; return x * y")); + assertEquals(4, exec("def x = (short)2; int y = (int)2; return x * y")); + assertEquals(4, exec("def x = (char)2; int y = (int)2; return x * y")); + assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; int y = (int)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; int y = (int)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; int y = (int)2; return x * y")); + + assertEquals(4L, exec("def x = (byte)2; long y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (short)2; long y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (char)2; long y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (int)2; long y = (long)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; long y = (long)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; long y = (long)2; return x * y")); + + assertEquals(4F, exec("def x = (byte)2; float y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (short)2; float y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (char)2; float y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (int)2; float y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (long)2; float y = (float)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; float y = (float)2; return x * y")); + + assertEquals(4D, exec("def x = (byte)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (short)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (char)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (int)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (long)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (float)2; double y = (double)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y")); + + assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y")); + assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y")); + assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y")); + assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y")); + assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y")); + assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y")); + assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java index f287b1e4cf4..5cf9c186fd6 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java @@ -22,6 +22,13 @@ package org.elasticsearch.painless; /** Tests for or operator across all types */ public class OrTests extends ScriptTestCase { + public void testBasics() throws Exception { + assertEquals(5 | 3, exec("return 5 | 3;")); + assertEquals(5L | 3, exec("return 5L | 3;")); + assertEquals(5 | 3L, exec("return 5 | 3L;")); + assertEquals(7, exec("short x = 5; byte y = 3; return x | y;")); + } + public void testInt() throws Exception { assertEquals(5 | 12, exec("int x = 5; int y = 12; return x | y;")); assertEquals(5 | -12, exec("int x = 5; int y = -12; return x | y;")); @@ -45,4 +52,160 @@ public class OrTests extends ScriptTestCase { assertEquals(5L | -12L, exec("return 5L | -12L;")); assertEquals(7L | 15L | 3L, exec("return 7L | 15L | 3L;")); } + + public void testIllegal() throws Exception { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; int y = 1; return x | y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; int y = 1; return x | y"); + }); + } + + public void testDef() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; def y = (byte)1; return x | y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; def y = (byte)1; return x | y"); + }); + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y")); + assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y")); + assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y")); + assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y")); + assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y")); + assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y")); + assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y")); + + assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y")); + + assertEquals(true, exec("def x = true; def y = true; return x | y")); + assertEquals(true, exec("def x = true; def y = false; return x | y")); + assertEquals(true, exec("def x = false; def y = true; return x | y")); + assertEquals(false, exec("def x = false; def y = false; return x | y")); + } + + public void testDefTypedLHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; def y = (byte)1; return x | y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; def y = (byte)1; return x | y"); + }); + assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("short x = (short)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("char x = (char)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("int x = (int)4; def y = (byte)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (byte)1; return x | y")); + + assertEquals(5, exec("byte x = (byte)4; def y = (short)1; return x | y")); + assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y")); + assertEquals(5, exec("char x = (char)4; def y = (short)1; return x | y")); + assertEquals(5, exec("int x = (int)4; def y = (short)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (short)1; return x | y")); + + assertEquals(5, exec("byte x = (byte)4; def y = (char)1; return x | y")); + assertEquals(5, exec("short x = (short)4; def y = (char)1; return x | y")); + assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y")); + assertEquals(5, exec("int x = (int)4; def y = (char)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (char)1; return x | y")); + + assertEquals(5, exec("byte x = (byte)4; def y = (int)1; return x | y")); + assertEquals(5, exec("short x = (short)4; def y = (int)1; return x | y")); + assertEquals(5, exec("char x = (char)4; def y = (int)1; return x | y")); + assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (int)1; return x | y")); + + assertEquals(5L, exec("byte x = (byte)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("short x = (short)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("char x = (char)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("int x = (int)4; def y = (long)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y")); + + assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y")); + assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y")); + assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y")); + assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y")); + assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y")); + + assertEquals(true, exec("boolean x = true; def y = true; return x | y")); + assertEquals(true, exec("boolean x = true; def y = false; return x | y")); + assertEquals(true, exec("boolean x = false; def y = true; return x | y")); + assertEquals(false, exec("boolean x = false; def y = false; return x | y")); + } + + public void testDefTypedRHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; byte y = (byte)1; return x | y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; byte y = (byte)1; return x | y"); + }); + assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x | y")); + assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y")); + assertEquals(5, exec("def x = (char)4; short y = (short)1; return x | y")); + assertEquals(5, exec("def x = (int)4; short y = (short)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x | y")); + assertEquals(5, exec("def x = (short)4; char y = (char)1; return x | y")); + assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y")); + assertEquals(5, exec("def x = (int)4; char y = (char)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x | y")); + assertEquals(5, exec("def x = (short)4; int y = (int)1; return x | y")); + assertEquals(5, exec("def x = (char)4; int y = (int)1; return x | y")); + assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x | y")); + + assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y")); + + assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y")); + assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y")); + assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y")); + assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y")); + assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y")); + + assertEquals(true, exec("def x = true; boolean y = true; return x | y")); + assertEquals(true, exec("def x = true; boolean y = false; return x | y")); + assertEquals(true, exec("def x = false; boolean y = true; return x | y")); + assertEquals(false, exec("def x = false; boolean y = false; return x | y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java similarity index 77% rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java index 54a36234fd8..9de618be4a7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java @@ -19,64 +19,7 @@ package org.elasticsearch.painless; -/** - * Tests binary operators across different types - */ -// TODO: NaN/Inf/overflow/... -public class BinaryOperatorTests extends ScriptTestCase { - - // TODO: move to per-type tests and test for each type - public void testBasics() { - assertEquals(2.25F / 1.5F, exec("return 2.25F / 1.5F;")); - assertEquals(2.25F % 1.5F, exec("return 2.25F % 1.5F;")); - assertEquals(2 - 1, exec("return 2 - 1;")); - assertEquals(1 << 2, exec("return 1 << 2;")); - assertEquals(4 >> 2, exec("return 4 >> 2;")); - assertEquals(-1 >>> 29, exec("return -1 >>> 29;")); - assertEquals(5 & 3, exec("return 5 & 3;")); - assertEquals(5 & 3L, exec("return 5 & 3L;")); - assertEquals(5L & 3, exec("return 5L & 3;")); - assertEquals(5 | 3, exec("return 5 | 3;")); - assertEquals(5L | 3, exec("return 5L | 3;")); - assertEquals(5 | 3L, exec("return 5 | 3L;")); - assertEquals(9 ^ 3, exec("return 9 ^ 3;")); - assertEquals(9L ^ 3, exec("return 9L ^ 3;")); - assertEquals(9 ^ 3L, exec("return 9 ^ 3L;")); - } - - public void testLongShifts() { - // note: we always promote the results of shifts too (unlike java) - assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;")); - assertEquals(1L << 2L, exec("long x = 1L; long y = 2L; return x << y;")); - assertEquals(4L >> 2L, exec("long x = 4L; long y = 2L; return x >> y;")); - assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;")); - assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;")); - assertEquals(-1L >>> 29L, exec("long x = -1L; long y = 29L; return x >>> y;")); - } - - public void testLongShiftsConst() { - assertEquals(1L << 2, exec("return 1L << 2;")); - assertEquals(1 << 2L, exec("return 1 << 2L;")); - assertEquals(4 >> 2L, exec("return 4 >> 2L;")); - assertEquals(4L >> 2, exec("return 4L >> 2;")); - assertEquals(-1L >>> 29, exec("return -1L >>> 29;")); - assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;")); - } - - public void testMixedTypes() { - assertEquals(8, exec("int x = 4; char y = 2; return x*y;")); - assertEquals(0.5, exec("double x = 1; float y = 2; return x / y;")); - assertEquals(1, exec("int x = 3; int y = 2; return x % y;")); - assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;")); - assertEquals(-1, exec("int x = 1; char y = 2; return x - y;")); - assertEquals(4, exec("int x = 1; char y = 2; return x << y;")); - assertEquals(-1, exec("int x = -1; char y = 29; return x >> y;")); - assertEquals(3, exec("int x = -1; char y = 30; return x >>> y;")); - assertEquals(1L, exec("int x = 5; long y = 3; return x & y;")); - assertEquals(7, exec("short x = 5; byte y = 3; return x | y;")); - assertEquals(10, exec("short x = 9; char y = 3; return x ^ y;")); - } - +public class PromotionTests extends ScriptTestCase { public void testBinaryPromotion() throws Exception { // byte/byte assertEquals((byte)1 + (byte)1, exec("byte x = 1; byte y = 1; return x+y;")); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java new file mode 100644 index 00000000000..69a81e0d77e --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +public class RegexTests extends ScriptTestCase { + public void testPatternAfterReturn() { + assertEquals(true, exec("return /foo/.matcher(\"foo\").matches()")); + assertEquals(false, exec("return /foo/.matcher(\"bar\").matches()")); + } + + public void testSlashesEscapePattern() { + assertEquals(true, exec("return /\\/\\//.matcher(\"//\").matches()")); + } + + public void testPatternAfterAssignment() { + assertEquals(true, exec("def a = /foo/; return a.matcher(\"foo\").matches()")); + } + + public void testPatternInIfStement() { + assertEquals(true, exec("if (/foo/.matcher(\"foo\").matches()) { return true } else { return false }")); + } + + public void testPatternAfterInfixBoolean() { + assertEquals(true, exec("return false || /foo/.matcher(\"foo\").matches()")); + assertEquals(true, exec("return true && /foo/.matcher(\"foo\").matches()")); + } + + public void testPatternAfterUnaryNotBoolean() { + assertEquals(false, exec("return !/foo/.matcher(\"foo\").matches()")); + assertEquals(true, exec("return !/foo/.matcher(\"bar\").matches()")); + } + + public void testInTernaryCondition() { + assertEquals(true, exec("return /foo/.matcher(\"foo\").matches() ? true : false")); + assertEquals(1, exec("def i = 0; i += /foo/.matcher(\"foo\").matches() ? 1 : 1; return i")); + } + + public void testInTernaryTrueArm() { + assertEquals(true, exec("def i = true; return i ? /foo/.matcher(\"foo\").matches() : false")); + } + + public void testInTernaryFalseArm() { + assertEquals(true, exec("def i = false; return i ? false : /foo/.matcher(\"foo\").matches()")); + } + + public void testRegexInFunction() { + assertEquals(true, exec("boolean m(String s) {/foo/.matcher(s).matches()} m(\"foo\")")); + } + + public void testReturnRegexFromFunction() { + assertEquals(true, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} m(true).matcher(\"foo\").matches()")); + } + +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java index 448d4262879..d2c9406c551 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java @@ -24,6 +24,11 @@ package org.elasticsearch.painless; public class RemainderTests extends ScriptTestCase { // TODO: byte,short,char + + public void testBasics() throws Exception { + assertEquals(2.25F % 1.5F, exec("return 2.25F % 1.5F;")); + assertEquals(1, exec("int x = 3; int y = 2; return x % y;")); + } public void testInt() throws Exception { assertEquals(1%1, exec("int x = 1; int y = 1; return x%y;")); @@ -132,4 +137,202 @@ public class RemainderTests extends ScriptTestCase { exec("return 1L%0L;"); }); } + + public void testDef() { + assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (short)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (char)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (int)2; def y = (byte)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (byte)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (byte)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (byte)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; def y = (short)2; return x % y")); + assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y")); + assertEquals(0, exec("def x = (char)2; def y = (short)2; return x % y")); + assertEquals(0, exec("def x = (int)2; def y = (short)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (short)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (short)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (short)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; def y = (char)2; return x % y")); + assertEquals(0, exec("def x = (short)2; def y = (char)2; return x % y")); + assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y")); + assertEquals(0, exec("def x = (int)2; def y = (char)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (char)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (char)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (char)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; def y = (int)2; return x % y")); + assertEquals(0, exec("def x = (short)2; def y = (int)2; return x % y")); + assertEquals(0, exec("def x = (char)2; def y = (int)2; return x % y")); + assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (int)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (int)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (int)2; return x % y")); + + assertEquals(0L, exec("def x = (byte)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (short)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (char)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (int)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (long)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (long)2; return x % y")); + + assertEquals(0F, exec("def x = (byte)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (short)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (char)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (int)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (long)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (float)2; return x % y")); + + assertEquals(0D, exec("def x = (byte)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (short)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (char)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (int)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (long)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y")); + assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y")); + assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y")); + } + + public void testDefTypedLHS() { + assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("short x = (short)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("char x = (char)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("int x = (int)2; def y = (byte)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (byte)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (byte)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (byte)2; return x % y")); + + assertEquals(0, exec("byte x = (byte)2; def y = (short)2; return x % y")); + assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y")); + assertEquals(0, exec("char x = (char)2; def y = (short)2; return x % y")); + assertEquals(0, exec("int x = (int)2; def y = (short)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (short)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (short)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (short)2; return x % y")); + + assertEquals(0, exec("byte x = (byte)2; def y = (char)2; return x % y")); + assertEquals(0, exec("short x = (short)2; def y = (char)2; return x % y")); + assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y")); + assertEquals(0, exec("int x = (int)2; def y = (char)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (char)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (char)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (char)2; return x % y")); + + assertEquals(0, exec("byte x = (byte)2; def y = (int)2; return x % y")); + assertEquals(0, exec("short x = (short)2; def y = (int)2; return x % y")); + assertEquals(0, exec("char x = (char)2; def y = (int)2; return x % y")); + assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (int)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (int)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (int)2; return x % y")); + + assertEquals(0L, exec("byte x = (byte)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("short x = (short)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("char x = (char)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("int x = (int)2; def y = (long)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (long)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (long)2; return x % y")); + + assertEquals(0F, exec("byte x = (byte)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("short x = (short)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("char x = (char)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("int x = (int)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("long x = (long)2; def y = (float)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (float)2; return x % y")); + + assertEquals(0D, exec("byte x = (byte)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("short x = (short)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("char x = (char)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("int x = (int)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("long x = (long)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("float x = (float)2; def y = (double)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y")); + + assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y")); + assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y")); + assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y")); + assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y")); + assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y")); + assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y")); + assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y")); + } + + public void testDefTypedRHS() { + assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (short)2; byte y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (char)2; byte y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (int)2; byte y = (byte)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; byte y = (byte)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; byte y = (byte)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; byte y = (byte)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; short y = (short)2; return x % y")); + assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y")); + assertEquals(0, exec("def x = (char)2; short y = (short)2; return x % y")); + assertEquals(0, exec("def x = (int)2; short y = (short)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; short y = (short)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; short y = (short)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; short y = (short)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; char y = (char)2; return x % y")); + assertEquals(0, exec("def x = (short)2; char y = (char)2; return x % y")); + assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y")); + assertEquals(0, exec("def x = (int)2; char y = (char)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; char y = (char)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; char y = (char)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; char y = (char)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; int y = (int)2; return x % y")); + assertEquals(0, exec("def x = (short)2; int y = (int)2; return x % y")); + assertEquals(0, exec("def x = (char)2; int y = (int)2; return x % y")); + assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; int y = (int)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; int y = (int)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; int y = (int)2; return x % y")); + + assertEquals(0L, exec("def x = (byte)2; long y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (short)2; long y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (char)2; long y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (int)2; long y = (long)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; long y = (long)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; long y = (long)2; return x % y")); + + assertEquals(0F, exec("def x = (byte)2; float y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (short)2; float y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (char)2; float y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (int)2; float y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (long)2; float y = (float)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; float y = (float)2; return x % y")); + + assertEquals(0D, exec("def x = (byte)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (short)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (char)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (int)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (long)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (float)2; double y = (double)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y")); + + assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y")); + assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y")); + assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y")); + assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y")); + assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y")); + assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y")); + assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java index 9136261c164..e1dbe9db0f7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java @@ -30,7 +30,7 @@ public class ReservedWordTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("int _score = 5; return _score;"); }); - assertTrue(expected.getMessage().contains("Variable name [_score] is reserved")); + assertTrue(expected.getMessage().contains("Variable [_score] is reserved")); } /** check that we can't write to _score, its read-only! */ @@ -46,7 +46,7 @@ public class ReservedWordTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("int doc = 5; return doc;"); }); - assertTrue(expected.getMessage().contains("Variable name [doc] is reserved")); + assertTrue(expected.getMessage().contains("Variable [doc] is reserved")); } /** check that we can't write to doc, its read-only! */ @@ -62,7 +62,7 @@ public class ReservedWordTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("int ctx = 5; return ctx;"); }); - assertTrue(expected.getMessage().contains("Variable name [ctx] is reserved")); + assertTrue(expected.getMessage().contains("Variable [ctx] is reserved")); } /** check that we can't write to ctx, its read-only! */ @@ -83,7 +83,7 @@ public class ReservedWordTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("int _value = 5; return _value;"); }); - assertTrue(expected.getMessage().contains("Variable name [_value] is reserved")); + assertTrue(expected.getMessage().contains("Variable [_value] is reserved")); } /** check that we can't write to _value, its read-only! */ diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index b4e416ca614..cb35b53db5c 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -76,7 +76,7 @@ public abstract class ScriptTestCase extends ESTestCase { */ public void assertBytecodeExists(String script, String bytecode) { final String asm = Debugger.toString(script); - assertTrue("bytecode not found", asm.contains(bytecode)); + assertTrue("bytecode not found, got: \n" + asm , asm.contains(bytecode)); } /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */ diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java new file mode 100644 index 00000000000..d409c8c5f7f --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java @@ -0,0 +1,547 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless; + +/** Tests for shift operator across all types */ +public class ShiftTests extends ScriptTestCase { + + public void testBasics() { + assertEquals(1 << 2, exec("return 1 << 2;")); + assertEquals(4 >> 2, exec("return 4 >> 2;")); + assertEquals(-1 >>> 29, exec("return -1 >>> 29;")); + assertEquals(4, exec("int x = 1; char y = 2; return x << y;")); + assertEquals(-1, exec("int x = -1; char y = 29; return x >> y;")); + assertEquals(3, exec("int x = -1; char y = 30; return x >>> y;")); + } + + public void testLongShifts() { + assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;")); + assertEquals(1 << 2L, exec("int x = 1; long y = 2L; return x << y;")); + assertEquals(4 >> 2L, exec("int x = 4; long y = 2L; return x >> y;")); + assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;")); + assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;")); + assertEquals(-1 >>> 29L, exec("int x = -1; long y = 29L; return x >>> y;")); + } + + public void testLongShiftsConst() { + assertEquals(1L << 2, exec("return 1L << 2;")); + assertEquals(1 << 2L, exec("return 1 << 2L;")); + assertEquals(4 >> 2L, exec("return 4 >> 2L;")); + assertEquals(4L >> 2, exec("return 4L >> 2;")); + assertEquals(-1L >>> 29, exec("return -1L >>> 29;")); + assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;")); + } + + public void testBogusShifts() { + expectScriptThrows(ClassCastException.class, ()-> { + exec("long x = 1L; float y = 2; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("int x = 1; double y = 2L; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("float x = 1F; int y = 2; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("double x = 1D; int y = 2L; return x << y;"); + }); + } + + public void testBogusShiftsConst() { + expectScriptThrows(ClassCastException.class, ()-> { + exec("return 1L << 2F;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("return 1L << 2.0;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("return 1F << 2;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("return 1D << 2L"); + }); + } + + public void testLshDef() { + assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y")); + assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y")); + assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y")); + } + + public void testLshDefTypedLHS() { + assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x << y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (short)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (short)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x << y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (char)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (char)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x << y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (int)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (int)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x << y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (long)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (long)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (long)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (long)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y")); + + assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y")); + assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y")); + assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y")); + assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y")); + assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y")); + } + + public void testLshDefTypedRHS() { + assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x << y")); + assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y")); + assertEquals(2, exec("def x = (char)1; short y = (short)1; return x << y")); + assertEquals(2, exec("def x = (int)1; short y = (short)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x << y")); + assertEquals(2, exec("def x = (short)1; char y = (char)1; return x << y")); + assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y")); + assertEquals(2, exec("def x = (int)1; char y = (char)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x << y")); + assertEquals(2, exec("def x = (short)1; int y = (int)1; return x << y")); + assertEquals(2, exec("def x = (char)1; int y = (int)1; return x << y")); + assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; long y = (long)1; return x << y")); + assertEquals(2, exec("def x = (short)1; long y = (long)1; return x << y")); + assertEquals(2, exec("def x = (char)1; long y = (long)1; return x << y")); + assertEquals(2, exec("def x = (int)1; long y = (long)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y")); + + assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y")); + assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y")); + assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y")); + assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y")); + assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y")); + } + + public void testRshDef() { + assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y")); + } + + public void testRshDefTypeLHS() { + assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y")); + assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y")); + assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y")); + assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y")); + assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y")); + } + + public void testRshDefTypedLHS() { + assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y")); + + assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y")); + assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y")); + assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y")); + assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y")); + assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y")); + } + + public void testUshDef() { + assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y")); + } + + public void testUshDefTypedLHS() { + assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >>> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >>> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >>> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >>> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y")); + + assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y")); + assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y")); + assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y")); + assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y")); + assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y")); + } + + public void testUshDefTypedRHS() { + assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y")); + + assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y")); + assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y")); + assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y")); + assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y")); + assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y")); + } + + public void testBogusDefShifts() { + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; def y = 2F; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; def y = 2D; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; def y = 2; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; def y = 2L; return x << y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; def y = 2F; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; def y = 2D; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; def y = 2; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; def y = 2L; return x >> y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; def y = 2F; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; def y = 2D; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; def y = 2; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; def y = 2L; return x >>> y;"); + }); + } + + public void testBogusDefShiftsTypedLHS() { + expectScriptThrows(ClassCastException.class, ()-> { + exec("long x = 1L; def y = 2F; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("int x = 1; def y = 2D; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("float x = 1F; def y = 2; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("double x = 1D; def y = 2L; return x << y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("long x = 1L; def y = 2F; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("int x = 1; def y = 2D; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("float x = 1F; def y = 2; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("double x = 1D; def y = 2L; return x >> y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("long x = 1L; def y = 2F; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("int x = 1; def y = 2D; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("float x = 1F; def y = 2; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("double x = 1D; def y = 2L; return x >>> y;"); + }); + } + + public void testBogusDefShiftsTypedRHS() { + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; float y = 2F; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; double y = 2D; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; int y = 2; return x << y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; long y = 2L; return x << y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; float y = 2F; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; double y = 2D; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; int y = 2; return x >> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; long y = 2L; return x >> y;"); + }); + + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1L; float y = 2F; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1; double y = 2D; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1F; int y = 2; return x >>> y;"); + }); + expectScriptThrows(ClassCastException.class, ()-> { + exec("def x = 1D; long y = 2L; return x >>> y;"); + }); + } + +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java index 12d275de6e5..f42dd5f7c98 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java @@ -182,4 +182,27 @@ public class StringTests extends ScriptTestCase { }); assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char].")); } + + public void testDefConcat() { + assertEquals("a" + (byte)2, exec("def x = 'a'; def y = (byte)2; return x + y")); + assertEquals("a" + (short)2, exec("def x = 'a'; def y = (short)2; return x + y")); + assertEquals("a" + (char)2, exec("def x = 'a'; def y = (char)2; return x + y")); + assertEquals("a" + 2, exec("def x = 'a'; def y = (int)2; return x + y")); + assertEquals("a" + 2L, exec("def x = 'a'; def y = (long)2; return x + y")); + assertEquals("a" + 2F, exec("def x = 'a'; def y = (float)2; return x + y")); + assertEquals("a" + 2D, exec("def x = 'a'; def y = (double)2; return x + y")); + assertEquals("ab", exec("def x = 'a'; def y = 'b'; return x + y")); + assertEquals((byte)2 + "a", exec("def x = 'a'; def y = (byte)2; return y + x")); + assertEquals((short)2 + "a", exec("def x = 'a'; def y = (short)2; return y + x")); + assertEquals((char)2 + "a", exec("def x = 'a'; def y = (char)2; return y + x")); + assertEquals(2 + "a", exec("def x = 'a'; def y = (int)2; return y + x")); + assertEquals(2L + "a", exec("def x = 'a'; def y = (long)2; return y + x")); + assertEquals(2F + "a", exec("def x = 'a'; def y = (float)2; return y + x")); + assertEquals(2D + "a", exec("def x = 'a'; def y = (double)2; return y + x")); + assertEquals("anull", exec("def x = 'a'; def y = null; return x + y")); + assertEquals("nullb", exec("def x = null; def y = 'b'; return x + y")); + expectScriptThrows(NullPointerException.class, () -> { + exec("def x = null; def y = null; return x + y"); + }); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java index f6a14b175e6..4817b040326 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java @@ -22,6 +22,11 @@ package org.elasticsearch.painless; /** Tests for subtraction operator across all types */ //TODO: NaN/Inf/overflow/... public class SubtractionTests extends ScriptTestCase { + + public void testBasics() throws Exception { + assertEquals(2 - 1, exec("return 2 - 1;")); + assertEquals(-1, exec("int x = 1; char y = 2; return x - y;")); + } public void testInt() throws Exception { assertEquals(1-1, exec("int x = 1; int y = 1; return x-y;")); @@ -176,4 +181,178 @@ public class SubtractionTests extends ScriptTestCase { assertEquals(10.0-0.0, exec("return 10.0-0.0;")); assertEquals(0.0-0.0, exec("return 0.0-0.0;")); } + + public void testDef() { + assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (short)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (char)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (int)1; def y = (byte)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; def y = (byte)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (byte)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (byte)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; def y = (short)1; return x - y")); + assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y")); + assertEquals(0, exec("def x = (char)1; def y = (short)1; return x - y")); + assertEquals(0, exec("def x = (int)1; def y = (short)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; def y = (short)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (short)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (short)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; def y = (char)1; return x - y")); + assertEquals(0, exec("def x = (short)1; def y = (char)1; return x - y")); + assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y")); + assertEquals(0, exec("def x = (int)1; def y = (char)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; def y = (char)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (char)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (char)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; def y = (int)1; return x - y")); + assertEquals(0, exec("def x = (short)1; def y = (int)1; return x - y")); + assertEquals(0, exec("def x = (char)1; def y = (int)1; return x - y")); + assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; def y = (int)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (int)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (int)1; return x - y")); + + assertEquals(0L, exec("def x = (byte)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (short)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (char)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (int)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (long)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (long)1; return x - y")); + + assertEquals(0F, exec("def x = (byte)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (short)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (char)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (int)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (long)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (float)1; return x - y")); + + assertEquals(0D, exec("def x = (byte)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (short)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (char)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (int)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y")); + } + + public void testDefTypedLHS() { + assertEquals(0, exec("byte x = (byte)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("short x = (short)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("char x = (char)1; def y = (byte)1; return x - y")); + assertEquals(0, exec("int x = (int)1; def y = (byte)1; return x - y")); + assertEquals(0L, exec("long x = (long)1; def y = (byte)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (byte)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (byte)1; return x - y")); + + assertEquals(0, exec("byte x = (byte)1; def y = (short)1; return x - y")); + assertEquals(0, exec("short x = (short)1; def y = (short)1; return x - y")); + assertEquals(0, exec("char x = (char)1; def y = (short)1; return x - y")); + assertEquals(0, exec("int x = (int)1; def y = (short)1; return x - y")); + assertEquals(0L, exec("long x = (long)1; def y = (short)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (short)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (short)1; return x - y")); + + assertEquals(0, exec("byte x = (byte)1; def y = (char)1; return x - y")); + assertEquals(0, exec("short x = (short)1; def y = (char)1; return x - y")); + assertEquals(0, exec("char x = (char)1; def y = (char)1; return x - y")); + assertEquals(0, exec("int x = (int)1; def y = (char)1; return x - y")); + assertEquals(0L, exec("long x = (long)1; def y = (char)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (char)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (char)1; return x - y")); + + assertEquals(0, exec("byte x = (byte)1; def y = (int)1; return x - y")); + assertEquals(0, exec("short x = (short)1; def y = (int)1; return x - y")); + assertEquals(0, exec("char x = (char)1; def y = (int)1; return x - y")); + assertEquals(0, exec("int x = (int)1; def y = (int)1; return x - y")); + assertEquals(0L, exec("long x = (long)1; def y = (int)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (int)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (int)1; return x - y")); + + assertEquals(0L, exec("byte x = (byte)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("short x = (short)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("char x = (char)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("int x = (int)1; def y = (long)1; return x - y")); + assertEquals(0L, exec("long x = (long)1; def y = (long)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (long)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (long)1; return x - y")); + + assertEquals(0F, exec("byte x = (byte)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("short x = (short)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("char x = (char)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("int x = (int)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("long x = (long)1; def y = (float)1; return x - y")); + assertEquals(0F, exec("float x = (float)1; def y = (float)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (float)1; return x - y")); + + assertEquals(0D, exec("byte x = (byte)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("short x = (short)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("char x = (char)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("int x = (int)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("long x = (long)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("float x = (float)1; def y = (double)1; return x - y")); + assertEquals(0D, exec("double x = (double)1; def y = (double)1; return x - y")); + } + + public void testDefTypedRHS() { + assertEquals(0, exec("def x = (byte)1; byte y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (short)1; byte y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (char)1; byte y = (byte)1; return x - y")); + assertEquals(0, exec("def x = (int)1; byte y = (byte)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; byte y = (byte)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; byte y = (byte)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; byte y = (byte)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; short y = (short)1; return x - y")); + assertEquals(0, exec("def x = (short)1; short y = (short)1; return x - y")); + assertEquals(0, exec("def x = (char)1; short y = (short)1; return x - y")); + assertEquals(0, exec("def x = (int)1; short y = (short)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; short y = (short)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; short y = (short)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; short y = (short)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; char y = (char)1; return x - y")); + assertEquals(0, exec("def x = (short)1; char y = (char)1; return x - y")); + assertEquals(0, exec("def x = (char)1; char y = (char)1; return x - y")); + assertEquals(0, exec("def x = (int)1; char y = (char)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; char y = (char)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; char y = (char)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; char y = (char)1; return x - y")); + + assertEquals(0, exec("def x = (byte)1; int y = (int)1; return x - y")); + assertEquals(0, exec("def x = (short)1; int y = (int)1; return x - y")); + assertEquals(0, exec("def x = (char)1; int y = (int)1; return x - y")); + assertEquals(0, exec("def x = (int)1; int y = (int)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; int y = (int)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; int y = (int)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; int y = (int)1; return x - y")); + + assertEquals(0L, exec("def x = (byte)1; long y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (short)1; long y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (char)1; long y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (int)1; long y = (long)1; return x - y")); + assertEquals(0L, exec("def x = (long)1; long y = (long)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; long y = (long)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; long y = (long)1; return x - y")); + + assertEquals(0F, exec("def x = (byte)1; float y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (short)1; float y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (char)1; float y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (int)1; float y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (long)1; float y = (float)1; return x - y")); + assertEquals(0F, exec("def x = (float)1; float y = (float)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; float y = (float)1; return x - y")); + + assertEquals(0D, exec("def x = (byte)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (short)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (char)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (int)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (long)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (float)1; double y = (double)1; return x - y")); + assertEquals(0D, exec("def x = (double)1; double y = (double)1; return x - y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java index e670e23925b..f3e8dc2f430 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java @@ -39,4 +39,63 @@ public class UnaryTests extends ScriptTestCase { assertEquals(1, exec("return -(-1);")); assertEquals(0, exec("return -0;")); } + + public void testPlus() { + assertEquals(-1, exec("byte x = (byte)-1; return +x")); + assertEquals(-1, exec("short x = (short)-1; return +x")); + assertEquals(65535, exec("char x = (char)-1; return +x")); + assertEquals(-1, exec("int x = -1; return +x")); + assertEquals(-1L, exec("long x = -1L; return +x")); + assertEquals(-1.0F, exec("float x = -1F; return +x")); + assertEquals(-1.0, exec("double x = -1.0; return +x")); + } + + public void testDefNot() { + assertEquals(~1, exec("def x = (byte)1; return ~x")); + assertEquals(~1, exec("def x = (short)1; return ~x")); + assertEquals(~1, exec("def x = (char)1; return ~x")); + assertEquals(~1, exec("def x = 1; return ~x")); + assertEquals(~1L, exec("def x = 1L; return ~x")); + } + + public void testDefNotTypedRet() { + assertEquals((double)~1, exec("def x = (byte)1; double y = ~x; return y;")); + assertEquals((float)~1, exec("def x = (short)1; float y = ~x; return y;")); + assertEquals((long)~1, exec("def x = (char)1; long y = ~x; return y;")); + assertEquals(~1, exec("def x = 1; int y = ~x; return y;")); + } + + public void testDefNeg() { + assertEquals(-1, exec("def x = (byte)1; return -x")); + assertEquals(-1, exec("def x = (short)1; return -x")); + assertEquals(-1, exec("def x = (char)1; return -x")); + assertEquals(-1, exec("def x = 1; return -x")); + assertEquals(-1L, exec("def x = 1L; return -x")); + assertEquals(-1.0F, exec("def x = 1F; return -x")); + assertEquals(-1.0, exec("def x = 1.0; return -x")); + } + + public void testDefNegTypedRet() { + assertEquals((double)-1, exec("def x = (byte)1; double y = -x; return y;")); + assertEquals((float)-1, exec("def x = (short)1; float y = -x; return y;")); + assertEquals((long)-1, exec("def x = (char)1; long y = -x; return y;")); + assertEquals(-1, exec("def x = 1; int y = -x; return y;")); + } + + public void testDefPlus() { + assertEquals(-1, exec("def x = (byte)-1; return +x")); + assertEquals(-1, exec("def x = (short)-1; return +x")); + assertEquals(65535, exec("def x = (char)-1; return +x")); + assertEquals(-1, exec("def x = -1; return +x")); + assertEquals(-1L, exec("def x = -1L; return +x")); + assertEquals(-1.0F, exec("def x = -1F; return +x")); + assertEquals(-1.0D, exec("def x = -1.0; return +x")); + } + + public void testDefPlusTypedRet() { + assertEquals((double)-1, exec("def x = (byte)-1; double y = +x; return y;")); + assertEquals((float)-1, exec("def x = (short)-1; float y = +x; return y;")); + assertEquals((long)65535, exec("def x = (char)-1; long y = +x; return y;")); + assertEquals(-1, exec("def x = -1; int y = +x; return y;")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java index ee88c0000bd..a9e03095297 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java @@ -22,6 +22,10 @@ package org.elasticsearch.painless; import java.lang.invoke.WrongMethodTypeException; import java.util.Arrays; import java.util.Collections; +import java.util.regex.PatternSyntaxException; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.containsString; public class WhenThingsGoWrongTests extends ScriptTestCase { public void testNullPointer() { @@ -197,4 +201,37 @@ public class WhenThingsGoWrongTests extends ScriptTestCase { exec("def x = new ArrayList(); x.add('foo'); return x['bogus'];"); }); } + + /** + * Makes sure that we present a useful error message with a misplaced right-curly. This is important because we do some funky things in + * the parser with right-curly brackets to allow statements to be delimited by them at the end of blocks. + */ + public void testRCurlyNotDelim() { + IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> { + // We don't want PICKY here so we get the normal error message + exec("def i = 1} return 1", emptyMap(), emptyMap(), null); + }); + assertEquals("invalid sequence of tokens near ['}'].", e.getMessage()); + } + + public void testCantUsePatternCompile() { + IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> { + exec("Pattern.compile(\"aa\")"); + }); + assertEquals("Unknown call [compile] with [1] arguments on type [Pattern].", e.getMessage()); + } + + public void testBadRegexPattern() { + PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> { + exec("/\\ujjjj/"); // Invalid unicode + }); + assertThat(e.getMessage(), containsString("Illegal Unicode escape sequence near index 2")); + assertThat(e.getMessage(), containsString("\\ujjjj")); + } + + public void testBadBoxingCast() { + expectScriptThrows(ClassCastException.class, () -> { + exec("BitSet bs = new BitSet(); bs.and(2);"); + }); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java index f5dd0a92011..1921539b0bf 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java @@ -22,6 +22,13 @@ package org.elasticsearch.painless; /** Tests for xor operator across all types */ public class XorTests extends ScriptTestCase { + public void testBasics() throws Exception { + assertEquals(9 ^ 3, exec("return 9 ^ 3;")); + assertEquals(9L ^ 3, exec("return 9L ^ 3;")); + assertEquals(9 ^ 3L, exec("return 9 ^ 3L;")); + assertEquals(10, exec("short x = 9; char y = 3; return x ^ y;")); + } + public void testInt() throws Exception { assertEquals(5 ^ 12, exec("int x = 5; int y = 12; return x ^ y;")); assertEquals(5 ^ -12, exec("int x = 5; int y = -12; return x ^ y;")); @@ -59,4 +66,160 @@ public class XorTests extends ScriptTestCase { assertEquals(true, exec("return false ^ true;")); assertEquals(false, exec("return false ^ false;")); } + + public void testIllegal() throws Exception { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; int y = 1; return x ^ y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; int y = 1; return x ^ y"); + }); + } + + public void testDef() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; def y = (byte)1; return x ^ y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; def y = (byte)1; return x ^ y"); + }); + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); + + assertEquals(false, exec("def x = true; def y = true; return x ^ y")); + assertEquals(true, exec("def x = true; def y = false; return x ^ y")); + assertEquals(true, exec("def x = false; def y = true; return x ^ y")); + assertEquals(false, exec("def x = false; def y = false; return x ^ y")); + } + + public void testDefTypedLHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("float x = (float)4; def y = (byte)1; return x ^ y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("double x = (double)4; def y = (byte)1; return x ^ y"); + }); + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y")); + + assertEquals(false, exec("def x = true; def y = true; return x ^ y")); + assertEquals(true, exec("def x = true; def y = false; return x ^ y")); + assertEquals(true, exec("def x = false; def y = true; return x ^ y")); + assertEquals(false, exec("def x = false; def y = false; return x ^ y")); + } + + public void testDefTypedRHS() { + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (float)4; byte y = (byte)1; return x ^ y"); + }); + expectScriptThrows(ClassCastException.class, () -> { + exec("def x = (double)4; byte y = (byte)1; return x ^ y"); + }); + assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; short y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; short y = (short)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; char y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; char y = (char)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; int y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; int y = (int)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x ^ y")); + + assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y")); + + assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y")); + assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y")); + assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y")); + assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y")); + assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y")); + + assertEquals(false, exec("def x = true; boolean y = true; return x ^ y")); + assertEquals(true, exec("def x = true; boolean y = false; return x ^ y")); + assertEquals(true, exec("def x = false; boolean y = true; return x ^ y")); + assertEquals(false, exec("def x = false; boolean y = false; return x ^ y")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java index 1a75c970fc4..d1852ada27b 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java @@ -78,4 +78,17 @@ public class ParserTests extends ScriptTestCase { exception = expectThrows(RuntimeException.class, () -> buildAntlrTree("((Map)x.-x)")); assertTrue(exception.getMessage().contains("unexpected character")); } + + public void testLambdaSyntax() { + buildAntlrTree("call(p -> {p.doSomething();});"); + buildAntlrTree("call(int p -> {p.doSomething();});"); + buildAntlrTree("call((p, u, v) -> {p.doSomething(); blah = 1;});"); + buildAntlrTree("call(1, (p, u, v) -> {p.doSomething(); blah = 1;}, 3);"); + buildAntlrTree("call((p, u, v) -> {p.doSomething(); blah = 1;});"); + buildAntlrTree("call(x, y, z, (int p, int u, int v) -> {p.doSomething(); blah = 1;});"); + buildAntlrTree("call(x, y, z, (long p, List u, String v) -> {p.doSomething(); blah = 1;});"); + buildAntlrTree("call(x, y, z, (int p, u, int v) -> {p.doSomething(); blah = 1;});"); + buildAntlrTree("call(x, (int p, u, int v) -> {p.doSomething(); blah = 1;}, z," + + " (int p, u, int v) -> {p.doSomething(); blah = 1;}, 'test');"); + } } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java index f9f1aa575b7..c451a245754 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java @@ -34,6 +34,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -192,6 +193,13 @@ public final class ExtractQueryTermsService { } else if (query instanceof BlendedTermQuery) { List terms = ((BlendedTermQuery) query).getTerms(); return new HashSet<>(terms); + } else if (query instanceof DisjunctionMaxQuery) { + List disjuncts = ((DisjunctionMaxQuery) query).getDisjuncts(); + Set terms = new HashSet<>(); + for (Query disjunct : disjuncts) { + terms.addAll(extractQueryTerms(disjunct)); + } + return terms; } else if (query instanceof SpanTermQuery) { return Collections.singleton(((SpanTermQuery) query).getTerm()); } else if (query instanceof SpanNearQuery) { diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 2f3108d6298..fbf06468e75 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -243,7 +243,7 @@ public final class PercolateQuery extends Query implements Accountable { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; + if (sameClassAs(o) == false) return false; PercolateQuery that = (PercolateQuery) o; @@ -254,7 +254,7 @@ public final class PercolateQuery extends Query implements Accountable { @Override public int hashCode() { - int result = super.hashCode(); + int result = classHash(); result = 31 * result + documentType.hashCode(); result = 31 * result + documentSource.hashCode(); return result; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index aadcabda006..9955375b183 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -447,7 +447,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder docs = doc.docs(); int rootDocIndex = docs.size() - 1; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index 41a43d39328..4a110172d77 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -28,10 +28,10 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.Highlighters; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.highlight.HighlightPhase; import org.elasticsearch.search.highlight.SearchContextHighlight; @@ -42,26 +42,27 @@ import org.elasticsearch.search.internal.SubSearchContext; import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Map; -// Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted, -// but the source of the PercolateQuery gets highlighted by each hit containing a query. -public class PercolatorHighlightSubFetchPhase implements FetchSubPhase { +/** + * Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted, + * but the source of the PercolateQuery gets highlighted by each hit containing a query. + */ +public final class PercolatorHighlightSubFetchPhase extends HighlightPhase { - private final HighlightPhase highlightPhase; - - @Inject - public PercolatorHighlightSubFetchPhase(HighlightPhase highlightPhase) { - this.highlightPhase = highlightPhase; + public PercolatorHighlightSubFetchPhase(Settings settings, Highlighters highlighters) { + super(settings, highlighters); } - @Override - public boolean hitsExecutionNeeded(SearchContext context) { + + boolean hitsExecutionNeeded(SearchContext context) { // for testing return context.highlight() != null && locatePercolatorQuery(context.query()) != null; } @Override public void hitsExecute(SearchContext context, InternalSearchHit[] hits) { + if (hitsExecutionNeeded(context) == false) { + return; + } PercolateQuery percolateQuery = locatePercolatorQuery(context.query()); if (percolateQuery == null) { // shouldn't happen as we checked for the existence of a percolator query in hitsExecutionNeeded(...) @@ -93,26 +94,12 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase { percolatorLeafReaderContext, 0, percolatorIndexSearcher ); hitContext.cache().clear(); - highlightPhase.hitExecute(subSearchContext, hitContext); + super.hitExecute(subSearchContext, hitContext); hit.highlightFields().putAll(hitContext.hit().getHighlightFields()); } } } - @Override - public Map parseElements() { - return Collections.emptyMap(); - } - - @Override - public boolean hitExecutionNeeded(SearchContext context) { - return false; - } - - @Override - public void hitExecute(SearchContext context, HitContext hitContext) { - } - static PercolateQuery locatePercolatorQuery(Query query) { if (query instanceof PercolateQuery) { return (PercolateQuery) query; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java index 963f14041b0..dfc11804dd9 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java @@ -28,15 +28,21 @@ import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.highlight.HighlightPhase; + +import java.util.Optional; public class PercolatorPlugin extends Plugin { public static final String NAME = "percolator"; private final boolean transportClientMode; + private final Settings settings; public PercolatorPlugin(Settings settings) { this.transportClientMode = transportClientMode(settings); + this.settings = settings; } @Override @@ -67,7 +73,7 @@ public class PercolatorPlugin extends Plugin { public void onModule(SearchModule module) { module.registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD); - module.registerFetchSubPhase(PercolatorHighlightSubFetchPhase.class); + module.registerFetchSubPhase(new PercolatorHighlightSubFetchPhase(settings, module.getHighlighters())); } public void onModule(SettingsModule module) { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java index 444e47d90ce..b9430a32425 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; @@ -362,6 +363,28 @@ public class ExtractQueryTermsServiceTests extends ESTestCase { assertThat(e.getUnsupportedQuery(), sameInstance(unsupportedQuery)); } + public void testExtractQueryMetadata_disjunctionMaxQuery() { + TermQuery termQuery1 = new TermQuery(new Term("_field", "_term1")); + TermQuery termQuery2 = new TermQuery(new Term("_field", "_term2")); + TermQuery termQuery3 = new TermQuery(new Term("_field", "_term3")); + TermQuery termQuery4 = new TermQuery(new Term("_field", "_term4")); + DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery( + Arrays.asList(termQuery1, termQuery2, termQuery3, termQuery4), 0.1f + ); + + List terms = new ArrayList<>(extractQueryTerms(disjunctionMaxQuery)); + Collections.sort(terms); + assertThat(terms.size(), equalTo(4)); + assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field())); + assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes())); + assertThat(terms.get(1).field(), equalTo(termQuery2.getTerm().field())); + assertThat(terms.get(1).bytes(), equalTo(termQuery2.getTerm().bytes())); + assertThat(terms.get(2).field(), equalTo(termQuery3.getTerm().field())); + assertThat(terms.get(2).bytes(), equalTo(termQuery3.getTerm().bytes())); + assertThat(terms.get(3).field(), equalTo(termQuery4.getTerm().field())); + assertThat(terms.get(3).bytes(), equalTo(termQuery4.getTerm().bytes())); + } + public void testCreateQueryMetadataQuery() throws Exception { MemoryIndex memoryIndex = new MemoryIndex(false); memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index c090ebf6dbf..67024150fb7 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -20,7 +20,15 @@ package org.elasticsearch.percolator; import com.fasterxml.jackson.core.JsonParseException; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -30,10 +38,14 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; import org.elasticsearch.index.query.QueryBuilder; @@ -44,11 +56,14 @@ import org.hamcrest.Matchers; import org.junit.BeforeClass; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; public class PercolateQueryBuilderTests extends AbstractQueryTestCase { @@ -233,6 +248,27 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase docs = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + docs.add(new ParseContext.Document()); + } + + Analyzer analyzer = new WhitespaceAnalyzer(); + ParsedDocument parsedDocument = new ParsedDocument(null, "_id", "_type", null, -1L, -1L, docs, null, null); + IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument); + assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs)); + + // ensure that any query get modified so that the nested docs are never included as hits: + Query query = new MatchAllDocsQuery(); + BooleanQuery result = (BooleanQuery) indexSearcher.createNormalizedWeight(query, true).getQuery(); + assertThat(result.clauses().size(), equalTo(2)); + assertThat(result.clauses().get(0).getQuery(), sameInstance(query)); + assertThat(result.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST)); + assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT)); + } + private static BytesReference randomSource() { try { XContentBuilder xContent = XContentFactory.jsonBuilder(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index 07959db1ff1..4879badc7d3 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -379,6 +379,16 @@ public class PercolateQueryTests extends ESTestCase { public String toString(String field) { return "custom{" + field + "}"; } + + @Override + public boolean equals(Object obj) { + return sameClassAs(obj); + } + + @Override + public int hashCode() { + return classHash(); + } } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index 3f7aaafc105..1dfd941e8e0 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -25,6 +25,8 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.Highlighters; import org.elasticsearch.search.highlight.SearchContextHighlight; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; @@ -44,16 +46,13 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase { Mockito.mock(IndexSearcher.class)) .build(); - PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(null); + PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(Settings.EMPTY, + new Highlighters(Settings.EMPTY)); SearchContext searchContext = Mockito.mock(SearchContext.class); Mockito.when(searchContext.highlight()).thenReturn(new SearchContextHighlight(Collections.emptyList())); Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery()); assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(false)); - IllegalStateException exception = expectThrows(IllegalStateException.class, - () -> subFetchPhase.hitsExecute(searchContext, null)); - assertThat(exception.getMessage(), equalTo("couldn't locate percolator query")); - Mockito.when(searchContext.query()).thenReturn(percolateQuery); assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(true)); } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index a277549f48e..e7690e23df4 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -18,12 +18,18 @@ */ package org.elasticsearch.percolator; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -32,6 +38,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.Collection; import java.util.Collections; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery; @@ -42,6 +49,7 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery; import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery; import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.instanceOf; @@ -382,4 +390,53 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query")); } + public void testPercolateQueryWithNestedDocuments() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject() + .startObject("employee").field("type", "nested").startObject("properties") + .startObject("name").field("type", "text").endObject().endObject().endObject().endObject() + .endObject(); + createIndex("test", client().admin().indices().prepareCreate("test") + .addMapping("employee", mapping) + .addMapping("queries", "query", "type=percolator") + ); + client().prepareIndex("test", "queries", "q").setSource(jsonBuilder().startObject() + .field("query", QueryBuilders.nestedQuery("employee", + QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg) + ).endObject()) + .setRefreshPolicy(IMMEDIATE) + .get(); + + SearchResponse response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "employee", + XContentFactory.jsonBuilder() + .startObject().field("companyname", "stark") + .startArray("employee") + .startObject().field("name", "virginia potts").endObject() + .startObject().field("name", "tony stark").endObject() + .endArray() + .endObject().bytes())) + .get(); + assertHitCount(response, 1); + assertThat(response.getHits().getAt(0).getId(), equalTo("q")); + + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "employee", + XContentFactory.jsonBuilder() + .startObject().field("companyname", "notstark") + .startArray("employee") + .startObject().field("name", "virginia stark").endObject() + .startObject().field("name", "tony stark").endObject() + .endArray() + .endObject().bytes())) + .get(); + assertHitCount(response, 0); + + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", "employee", + XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes())) + .get(); + assertHitCount(response, 0); + } + } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java index 3aea4dbce8a..284e51e054f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -77,6 +77,8 @@ public abstract class AbstractBaseReindexRestHandler< action.execute(internal, new BulkIndexByScrollResponseContentListener<>(channel, params)); return; + } else { + internal.setShouldPersistResult(true); } /* diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 7505f490f45..80a6ff891da 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -93,6 +93,11 @@ public abstract class AbstractBulkByScrollRequest indexingFailures = new ArrayList<>(indexingFailuresCount); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java index 382f5b51726..9841794ca2a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java @@ -19,7 +19,10 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; @@ -30,14 +33,18 @@ import org.elasticsearch.rest.action.support.RestToXContentListener; import org.elasticsearch.tasks.TaskId; import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.nodeSettingListener; public class RestRethrottleAction extends BaseRestHandler { private final TransportRethrottleAction action; + private final ClusterService clusterService; @Inject - public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action) { + public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action, + ClusterService clusterService) { super(settings, client); this.action = action; + this.clusterService = clusterService; controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this); controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this); controller.registerHandler(POST, "/_reindex/{taskId}/_rethrottle", this); @@ -52,6 +59,7 @@ public class RestRethrottleAction extends BaseRestHandler { throw new IllegalArgumentException("requests_per_second is a required parameter"); } internalRequest.setRequestsPerSecond(requestsPerSecond); - action.execute(internalRequest, new RestToXContentListener<>(channel)); + ActionListener listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel)); + action.execute(internalRequest, listener); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index d083f7b2b14..b716050c036 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.tasks.TransportTasksAction; import org.elasticsearch.cluster.ClusterName; @@ -31,6 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index 2bb7fd627e4..dcaa80a8169 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -31,6 +30,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.Engine.Operation.Origin; import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.ingest.IngestTestPlugin; import org.junit.BeforeClass; diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml index bdad5f581bc..13a98921c05 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml @@ -69,10 +69,13 @@ - is_false: deleted - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task - is_false: node_failures + # The task will be in the response even if it finished before we got here + # because of task persistence. + - is_true: task --- "Response for version conflict": diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml index 0ff382ff751..65a22781550 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml @@ -134,7 +134,7 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task @@ -197,6 +197,6 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml index f4025383321..b54d8bd0e95 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml @@ -93,10 +93,13 @@ - is_false: deleted - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task - is_false: node_failures + # The task will be in the response even if it finished before we got here + # because of task persistence. + - is_true: task --- "Response format for version conflict": diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml index 597cfa4240f..73e1a3a3a94 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml @@ -156,7 +156,7 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task @@ -214,6 +214,6 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml index 843bb9b6eb5..8ba4013caa3 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml @@ -53,10 +53,13 @@ - is_false: deleted - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task - is_false: node_failures + # The task will be in the response even if it finished before we got here + # because of task persistence. + - is_true: task --- "Response for version conflict": diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml index 3179ce32a19..e13e29bc3f5 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml @@ -122,7 +122,7 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task @@ -172,6 +172,6 @@ task_id: $task - do: - tasks.list: + task.get: wait_for_completion: true task_id: $task diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1 deleted file mode 100644 index 95dab25e74a..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da08d9919f54efd2e09968d49fe05f6ce3f0c7ce \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..013def114d4 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +9cd8cbea5baef18a36bee86846a9ba026d2a02e0 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1 deleted file mode 100644 index 70f83bf52cc..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77905f563c47994a764a6ab3d5ec198c174567a7 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..12c861c24ab --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +86c6d6a367ed658351bd8c8828d6ed647ac79b7e \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1 deleted file mode 100644 index 8e2f7ab8b98..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3ee5d909c269e5da7a92715f41ead88943b38123 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..6f571d75537 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +6553bf764a69cd15e4fe1e55661382872795b853 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1 deleted file mode 100644 index 981855d5a97..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b7bdbf9efa84f8d8875bd7f1d8734276930b9c3 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..2ea2d6b9622 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +979817950bc806400d8fa12a609ef215b5bdebd6 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1 deleted file mode 100644 index 4ff0afee687..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e80e912621276e1009b72c06d5def188976c5426 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1 new file mode 100644 index 00000000000..6677cfd3fc4 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1 @@ -0,0 +1 @@ +2a720b647b6a202ec1d8d91db02006ae9539670b \ No newline at end of file diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle index 93d361c989c..fbce12f8126 100644 --- a/qa/backwards-5.0/build.gradle +++ b/qa/backwards-5.0/build.gradle @@ -18,6 +18,6 @@ integTest { cluster { numNodes = 2 numBwcNodes = 1 - bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC + bwcVersion = "5.0.0-alpha4-SNAPSHOT" // this is the same as the current version until we released the first RC } } diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 5c766018f13..fdf52da2634 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -307,6 +307,12 @@ public class InstallPluginCommandTests extends ESTestCase { assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); } + public void testUnknownPlugin() throws Exception { + Tuple env = createEnv(fs, temp); + UserError e = expectThrows(UserError.class, () -> installPlugin("foo", env.v1())); + assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo")); + } + public void testPluginsDirMissing() throws Exception { Tuple env = createEnv(fs, temp); Files.delete(env.v2().pluginsFile()); diff --git a/qa/smoke-test-reindex-with-groovy/build.gradle b/qa/smoke-test-reindex-with-painless/build.gradle similarity index 92% rename from qa/smoke-test-reindex-with-groovy/build.gradle rename to qa/smoke-test-reindex-with-painless/build.gradle index c4b462ce45a..51a2647f444 100644 --- a/qa/smoke-test-reindex-with-groovy/build.gradle +++ b/qa/smoke-test-reindex-with-painless/build.gradle @@ -18,9 +18,3 @@ */ apply plugin: 'elasticsearch.rest-test' - -integTest { - cluster { - setting 'script.inline', 'true' - } -} diff --git a/qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java similarity index 89% rename from qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java rename to qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java index 975bf596eaa..7857259a60d 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java +++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java @@ -27,8 +27,8 @@ import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -public class SmokeTestReindexWithGroovyIT extends ESRestTestCase { - public SmokeTestReindexWithGroovyIT(@Name("yaml") RestTestCandidate testCandidate) { +public class SmokeTestReindexWithPainlessIT extends ESRestTestCase { + public SmokeTestReindexWithPainlessIT(@Name("yaml") RestTestCandidate testCandidate) { super(testCandidate); } diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml similarity index 97% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml index 783838d7f8e..0e99d862fd1 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml @@ -18,6 +18,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx._source.user = "other" + ctx._source.user - match: {created: 1} - match: {noops: 0} @@ -57,6 +58,7 @@ dest: index: new_twitter script: + lang: painless inline: if (ctx._id == "1") {ctx._source.user = "other" + ctx._source.user} - match: {created: 2} - match: {noops: 0} @@ -116,6 +118,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx._parent = ctx._source.user - match: {created: 1} - match: {noops: 0} @@ -159,6 +162,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx._routing = ctx._source.user - match: {created: 2} - match: {noops: 0} @@ -217,6 +221,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx._parent = ctx._source.user; ctx._routing = "cat" - match: {created: 1} - match: {noops: 0} @@ -262,6 +267,7 @@ dest: index: new_twitter script: + lang: painless inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"} - match: {created: 1} - match: {noops: 1} @@ -314,6 +320,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx.op = "noop" - match: {updated: 0} - match: {noops: 2} @@ -354,6 +361,7 @@ index: new_twitter version_type: external script: + lang: painless inline: ctx._source.user = "other" + ctx._source.user; ctx._version = null - match: {updated: 1} - match: {noops: 0} @@ -393,6 +401,7 @@ dest: index: new_twitter script: + lang: painless inline: ctx._source.user = "other" + ctx._source.user; ctx._id = null - match: {created: 1} - match: {noops: 0} @@ -432,6 +441,7 @@ dest: index: new_twitter script: + lang: painless inline: if (ctx._source.user == "kimchy") {ctx._index = 'other_new_twitter'} - match: {created: 2} - match: {noops: 0} @@ -504,6 +514,7 @@ index: index2 type: type2 script: + lang: painless inline: "ctx._id = ctx._source.lang + '_' + ctx._source.id; if (ctx._source.lang != \"en\" ) {ctx.op = 'delete'}" - match: {created: 1} diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml similarity index 80% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml index 6fba2c9bd49..5ec35b4c9dc 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml @@ -19,5 +19,6 @@ dest: index: new_twitter script: + lang: painless inline: syntax errors are fun! - - match: {error.reason: 'Failed to compile inline script [syntax errors are fun!] using lang [groovy]'} + - match: {error.reason: 'compile error'} diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml similarity index 75% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml index ddd22246717..df514c61ceb 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml @@ -1,5 +1,8 @@ --- "Timeout": + - skip: + version: "all" + reason: painless doesn't support thread.sleep so we need to come up with a better way - do: index: index: twitter @@ -19,9 +22,10 @@ timeout: 10 query: script: + lang: painless # Sleep 100x longer than the timeout. That should cause a timeout! # Return true causes the document to try to be collected which is what actually triggers the timeout. - script: sleep(1000); return true + script: java.lang.Thread.sleep(1000); return true dest: index: new_twitter - is_true: timed_out diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml similarity index 62% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml index c43374e7de2..17e915fdc19 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml @@ -17,7 +17,8 @@ index: source query: script: - script: 1/0 # Divide by 0 to cause a search time exception + lang: painless + script: throw new IllegalArgumentException("Cats!") dest: index: dest - match: {created: 0} @@ -27,8 +28,8 @@ - is_true: failures.0.shard - match: {failures.0.index: source} - is_true: failures.0.node - - match: {failures.0.reason.type: general_script_exception} - - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"} - - match: {failures.0.reason.caused_by.type: arithmetic_exception} - - match: {failures.0.reason.caused_by.reason: Division by zero} + - match: {failures.0.reason.type: script_exception} + - match: {failures.0.reason.reason: runtime error} + - match: {failures.0.reason.caused_by.type: illegal_argument_exception} + - match: {failures.0.reason.caused_by.reason: Cats!} - gte: { took: 0 } diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml similarity index 92% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml index e4fef86b1d1..a4580b97334 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml @@ -15,6 +15,7 @@ refresh: true body: script: + lang: painless inline: ctx._source.user = "not" + ctx._source.user - match: {updated: 1} - match: {noops: 0} @@ -51,6 +52,7 @@ index: twitter body: script: + lang: painless inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"} - match: {updated: 1} - match: {noops: 1} @@ -96,6 +98,7 @@ index: twitter body: script: + lang: painless inline: ctx.op = "noop" - match: {updated: 0} - match: {noops: 2} @@ -118,6 +121,7 @@ index: twitter body: script: + lang: painless inline: ctx.junk = "stuff" --- @@ -137,6 +141,7 @@ index: twitter body: script: + lang: painless inline: ctx._id = "stuff" --- @@ -174,6 +179,7 @@ index: twitter body: script: + lang: painless inline: if (ctx._source.level != 11) {ctx._source.last_updated = "2016-01-02T00:00:00Z"} else {ctx.op = "delete"} - match: {updated: 3} - match: {deleted: 1} @@ -237,17 +243,16 @@ index: twitter body: script: - inline: "switch (ctx._source.level % 3) { - case 0: - ctx._source.last_updated = \"2016-01-02T00:00:00Z\"; - break; - case 1: - ctx.op = \"noop\"; - break; - case 2: - ctx.op = \"delete\"; - break; - }" + lang: painless + inline: | + int choice = ctx._source.level % 3; + if (choice == 0) { + ctx._source.last_updated = "2016-01-02T00:00:00Z"; + } else if (choice == 1) { + ctx.op = "noop"; + } else { + ctx.op = "delete"; + } - match: {updated: 2} - match: {deleted: 1} - match: {noops: 1} @@ -303,6 +308,7 @@ index: twitter body: script: + lang: painless inline: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"} - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' } diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml similarity index 78% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml index 66f9cfe5456..aa497bff77e 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml @@ -16,5 +16,6 @@ refresh: true body: script: + lang: painless inline: syntax errors are fun! - - match: {error.reason: 'Failed to compile inline script [syntax errors are fun!] using lang [groovy]'} + - match: {error.reason: 'compile error'} diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml similarity index 74% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml index e6bdd73757b..ac1ed14a1f8 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml @@ -1,5 +1,8 @@ --- "Timeout": + - skip: + version: "all" + reason: painless doesn't support thread.sleep so we need to come up with a better way - do: index: index: twitter @@ -18,9 +21,10 @@ body: query: script: + lang: painless # Sleep 100x longer than the timeout. That should cause a timeout! # Return true causes the document to try to be collected which is what actually triggers the timeout. - script: sleep(1000); return true + script: java.lang.Thread.sleep(1000); return true - is_true: timed_out - match: {updated: 0} - match: {noops: 0} diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml similarity index 58% rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml index 3aecca73332..d1ba9808fba 100644 --- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml +++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml @@ -16,15 +16,16 @@ body: query: script: - script: 1/0 # Divide by 0 to cause a search time exception + lang: painless + script: throw new IllegalArgumentException("Cats!") - match: {updated: 0} - match: {version_conflicts: 0} - match: {batches: 0} - is_true: failures.0.shard - match: {failures.0.index: source} - is_true: failures.0.node - - match: {failures.0.reason.type: general_script_exception} - - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"} - - match: {failures.0.reason.caused_by.type: arithmetic_exception} - - match: {failures.0.reason.caused_by.reason: Division by zero} + - match: {failures.0.reason.type: script_exception} + - match: {failures.0.reason.reason: runtime error} + - match: {failures.0.reason.caused_by.type: illegal_argument_exception} + - match: {failures.0.reason.caused_by.reason: Cats!} - gte: { took: 0 } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash index ac54bf3b664..c81f84cb778 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash @@ -248,10 +248,7 @@ fi } @test "[$GROUP] check lang-painless module" { - # we specify the version on the asm-5.0.4.jar so that the test does - # not spuriously pass if the jar is missing but the other asm jars - # are present - check_secure_module lang-painless antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar + check_secure_module lang-painless antlr4-runtime-*.jar asm-debug-all-*.jar } @test "[$GROUP] install javascript plugin" { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json index 87a86fb1298..0344702ecfe 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json @@ -20,6 +20,10 @@ "type" : "enum", "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"], "description" : "Search operation type" + }, + "max_concurrent_searches" : { + "type" : "number", + "description" : "Controls the maximum number of concurrent searches the multi search api will execute" } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json new file mode 100644 index 00000000000..8024f015e96 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json @@ -0,0 +1,23 @@ +{ + "task.get": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html", + "methods": ["GET"], + "url": { + "path": "/_tasks/{task_id}", + "paths": ["/_tasks/{task_id}"], + "parts": { + "task_id": { + "type": "string", + "description": "Return the task with specified id (node_id:task_number)" + } + }, + "params": { + "wait_for_completion": { + "type": "boolean", + "description": "Wait for the matching tasks to complete (default: false)" + } + } + }, + "body": null + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json index d9615a7fdab..a1913fbfc17 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json @@ -5,12 +5,7 @@ "url": { "path": "/_tasks", "paths": ["/_tasks", "/_tasks/{task_id}"], - "parts": { - "task_id": { - "type": "string", - "description": "Return the task with specified id (node_id:task_number)" - } - }, + "parts": {}, "params": { "node_id": { "type": "list", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml new file mode 100644 index 00000000000..5443059135a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml @@ -0,0 +1,75 @@ +--- +"Sliced scroll": + - do: + indices.create: + index: test_sliced_scroll + + - do: + index: + index: test_sliced_scroll + type: test + id: 42 + body: { foo: 1 } + + - do: + indices.refresh: {} + + - do: + search: + index: test_sliced_scroll + size: 1 + scroll: 1m + sort: foo + body: + slice: { + id: 0, + max: 3 + } + query: + match_all: {} + + - set: {_scroll_id: scroll_id} + + - do: + clear_scroll: + scroll_id: $scroll_id + + - do: + catch: /query_phase_execution_exception.*The number of slices.*index.max_slices_per_scroll/ + search: + index: test_sliced_scroll + size: 1 + scroll: 1m + body: + slice: { + id: 0, + max: 1025 + } + query: + match_all: {} + + - do: + indices.put_settings: + index: test_sliced_scroll + body: + index.max_slices_per_scroll: 1025 + + - do: + search: + index: test_sliced_scroll + size: 1 + scroll: 1m + body: + slice: { + id: 0, + max: 1025 + } + query: + match_all: {} + + - set: {_scroll_id: scroll_id} + + - do: + clear_scroll: + scroll_id: $scroll_id + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml new file mode 100644 index 00000000000..ba90e1541fe --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml @@ -0,0 +1,10 @@ +--- +"get task test": + # Note that this gets much better testing in reindex's tests because it actually saves the task + - do: + cluster.state: {} + + - do: + catch: missing + task.get: + task_id: foo:1 diff --git a/settings.gradle b/settings.gradle index 457e89c906f..82cfa58af62 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,6 +5,7 @@ List projects = [ 'rest-api-spec', 'core', 'docs', + 'benchmarks', 'distribution:integ-test-zip', 'distribution:zip', 'distribution:tar', @@ -46,11 +47,11 @@ List projects = [ 'qa:backwards-5.0', 'qa:evil-tests', 'qa:smoke-test-client', - 'qa:smoke-test-multinode', - 'qa:smoke-test-reindex-with-groovy', - 'qa:smoke-test-plugins', 'qa:smoke-test-ingest-with-all-dependencies', 'qa:smoke-test-ingest-disabled', + 'qa:smoke-test-multinode', + 'qa:smoke-test-plugins', + 'qa:smoke-test-reindex-with-painless', 'qa:vagrant', ] diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java index 629a5df3fee..1ed90408f7d 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java +++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java @@ -28,10 +28,8 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.AggregatorParsers; -import org.elasticsearch.search.dfs.DfsPhase; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.suggest.Suggesters; import org.elasticsearch.threadpool.ThreadPool; @@ -84,10 +82,9 @@ public class MockSearchService extends SearchService { @Inject public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService, - BigArrays bigArrays, DfsPhase dfsPhase, QueryPhase queryPhase, FetchPhase fetchPhase, - AggregatorParsers aggParsers, Suggesters suggesters) { - super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, dfsPhase, - queryPhase, fetchPhase, aggParsers, suggesters); + BigArrays bigArrays, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) { + super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, + fetchPhase, aggParsers, suggesters); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java index 4d5ff10f9cd..e1d6ab6dfaa 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java @@ -48,7 +48,7 @@ public abstract class AbstractTermsTestCase extends ESIntegTestCase { .addAggregation(terms("terms") .executionHint(randomExecutionHint()) .field(fieldName) - .size(0) + .size(10000) .collectMode(randomFrom(SubAggCollectionMode.values()))) .get(); assertSearchResponse(allTerms); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 2a3eecf4cc8..4e066bc7635 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -108,6 +108,7 @@ import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.store.IndicesStore; @@ -1633,7 +1634,11 @@ public abstract class ESIntegTestCase extends ESTestCase { .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b") .put("script.stored", "true") .put("script.inline", "true") - // wait short time for other active shards before actually deleting, default 30s not needed in tests + // by default we never cache below 10k docs in a segment, + // bypass this limit so that caching gets some testing in + // integration tests that usually create few documents + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), nodeOrdinal % 2 == 0) + // wait short time for other active shards before actually deleting, default 30s not needed in tests .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)); return builder.build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 1217662480a..0dee5f75948 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1773,7 +1773,7 @@ public final class InternalTestCluster extends TestCluster { OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node); while (true) { String routing = RandomStrings.randomAsciiOfLength(random, 10); - final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), type, null, routing).shardId().getId(); + final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), null, routing).shardId().getId(); if (shard == targetShard) { return routing; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java index 66167bc7ec8..3fc924176f0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java @@ -33,37 +33,35 @@ public class FakeRestRequest extends RestRequest { private final BytesReference content; + private final Method method; + + private final String path; + public FakeRestRequest() { - this(new HashMap<>()); + this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/"); } - public FakeRestRequest(Map headers) { - this(headers, new HashMap<>()); - } - - public FakeRestRequest(Map headers, Map params) { - this(headers, params, null); - } - - public FakeRestRequest(Map headers, Map params, BytesReference content) { + private FakeRestRequest(Map headers, Map params, BytesReference content, Method method, String path) { this.headers = headers; this.params = params; this.content = content; + this.method = method; + this.path = path; } @Override public Method method() { - return Method.GET; + return method; } @Override public String uri() { - return "/"; + return path; } @Override public String rawPath() { - return "/"; + return path; } @Override @@ -109,4 +107,46 @@ public class FakeRestRequest extends RestRequest { public Map params() { return params; } + + public static class Builder { + private Map headers = new HashMap<>(); + + private Map params = new HashMap<>(); + + private BytesReference content; + + private String path = "/"; + + private Method method = Method.GET; + + public Builder withHeaders(Map headers) { + this.headers = headers; + return this; + } + + public Builder withParams(Map params) { + this.params = params; + return this; + } + + public Builder withContent(BytesReference content) { + this.content = content; + return this; + } + + public Builder withPath(String path) { + this.path = path; + return this; + } + + public Builder withMethod(Method method) { + this.method = method; + return this; + } + + public FakeRestRequest build() { + return new FakeRestRequest(headers, params, content, method, path); + } + } + }